elasticsearch_record 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6cf2af35e8ed5eb94892c5e1c6c5a7b11fab36539a48a3106b3444862c3654fc
-  data.tar.gz: 63f47637943c1e65e8d2a53c9f887b57ca5bdb139c4b6575997abbd640f2586d
+  metadata.gz: 8c479344b27eac6994fc17c6607f42221b093ddfc5f6db36b88994f25f2a0e56
+  data.tar.gz: 884ba24246d3ab9ef3500546bca84dcb3eaea184ac20bcfde6a2713e0a6def79
 SHA512:
-  metadata.gz: 96f3ce41b50dc97f5e4d9cdf79bf5b58af5bedcf219edee5d5d22a88b6517efcad0f3d7a0e26ca0042493d83a59d274a3f33d80ec4e16032c7ec50dfcb6790b2
-  data.tar.gz: c041e7d0bc2e00dd006d1c5b87e403b03f2e9c98a55672f636b29821a318425dc72aabff46e439a4c1281c914f1a07ac86487961f06930fac4169d538a2b9888
+  metadata.gz: 554ca0302d07210beef1f8bcb4b49f65b9e734122ccbbfb4a44d619d7697d1e5ec4b4164890beacaab4106654e5cdb7c817adf81d5953d46ed9f600b395e8018
+  data.tar.gz: 2a6a3579a00f2734c623f364a1eea5ba9802896a122c74ea4503764c63ad0294f16279b02cf3ce0f6a8d924ef3cb15004c2e2a933e90f22d311f4cbae067597c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    elasticsearch_record (1.1.0)
+    elasticsearch_record (1.2.0)
       activerecord (~> 7.0.0)
       elasticsearch (~> 8.4)
 

data/README.md

@@ -240,20 +240,24 @@ total = scope.total
 - index_base_name
 - relay_id_attribute
 
-### Useful model instance methods
+### Useful model class methods
+- auto_increment?
+- max_result_window
 - source_column_names
 - searchable_column_names
 - find_by_query
 - msearch
 
-## ActiveRecord ConnectionAdapters schema-methods
+## ActiveRecord ConnectionAdapters table-methods
 Access these methods through the model's connection.
+
 ```ruby
   # returns mapping of provided table (index)
   model.connection.table_mappings('table-name')
 ```
 
 - table_mappings
+- table_metas
 - table_settings
 - table_aliases
 - table_state
@@ -261,12 +265,14 @@ Access these methods through the model's connection.
 - alias_exists?
 - setting_exists?
 - mapping_exists?
+- meta_exists?
 - max_result_window
 - cluster_info
 
 ## Active Record Schema migration methods
 Access these methods through the model's connection or within any `Migration`.
 
+**cluster actions:**
 - open_table
 - open_tables
 - close_table
@@ -274,8 +280,15 @@ Access these methods through the model's connection or within any `Migration`.
 - truncate_table
 - truncate_tables
 - drop_table
+- block_table
+- unblock_table
+- clone_table
 - create_table
 - change_table
+
+**table actions:**
+- change_meta
+- delete_meta
 - add_mapping
 - change_mapping
 - change_mapping_meta
@@ -299,11 +312,15 @@ class AddTests < ActiveRecord::Migration[7.0]
       t.setting :number_of_shards, "1"
       t.setting :number_of_replicas, 0
     end
+
+    # changes the auto-increment value
+    change_meta "assignments", :auto_increment, 3625
     
     create_table "settings", force: true do |t|
       t.mapping :created_at, :date
-      t.mapping :key, :keyword do |m|
+      t.mapping :key, :integer do |m|
         m.primary_key = true
+        m.auto_increment = 10
       end
       t.mapping :status, :keyword
       t.mapping :updated_at, :date
@@ -315,11 +332,11 @@ class AddTests < ActiveRecord::Migration[7.0]
     end
 
     add_mapping "settings", :active, :boolean do |m|
-      m.comment = "Hans"
+      m.comment = "Contains the active state"
     end
 
     change_table 'settings', force: true do |t|
-      t.add_setting( "index.search.idle.after", "20s")
+      t.add_setting("index.search.idle.after", "20s")
       t.add_setting("index.shard.check_on_startup", true)
       t.add_alias('supersettings')
     end
@@ -337,9 +354,6 @@ class AddTests < ActiveRecord::Migration[7.0]
       t.timestamps
     end
 
-    change_mapping_meta "vintage", :number, auto_increment: 'true'
-    change_mapping_meta "vintage", :number, peter: 'hans'
-
     change_mapping_attributes "vintage", :number, fields: {raw: {type: :keyword}}
   end
 

data/docs/CHANGELOG.md

@@ -1,6 +1,25 @@
 # ElasticsearchRecord - CHANGELOG
 
-## [1.1.0] - 2022-12-02
+## [1.2.0] - 2022-12-02
+* [add] `ElasticsearchRecord::SchemaMigration` to fix connection-related differences (like table_name_prefix, table_name_suffix)
+* [add] connection (config-related) 'table_name_prefix' & 'table_name_suffix' - now will be forwarded to all related models & schema-tables
+* [add] `#block_table`, `#unblock_table`, `#clone_table`, `#table_metas`, `#meta_exists?`, `#change_meta`, `#delete_meta` methods for Elasticsearch ConnectionAdapter
+* [add] `ElasticsearchRecord::Base.auto_increment?`
+* [add] index 'meta' method to access the `_meta` mapping
+* [add] `.ElasticsearchRecord::Base.relay_id_attribute` to relay a possible existing 'id'-attribute
+* [add] new enabled attribute `enabled` - which defines 'searchable attributes & fields' and gets also read from the index-mappings
+* [ref] insert a new record with primary_key & auto_increment through a wrapper `_insert_with_auto_increment`
+* [ref] resolve `primary_keys` now from the index `_meta` mapping first (old mapping-related 'meta.primary_key:"true"' is still supported)
+* [ref] disable 'strict' mode (= validation) of settings, alias, mappings as default (this can be still used with `strict: true`)
+* [ref] silent unsupported methods 'create/drop' for `ElasticsearchRecord::Tasks::ElasticsearchDatabaseTasks`
+* [ref] primary_key & auto_increment handling of custom defined mappings - now uses the index `_meta` mapping
+* [fix] creating a record with different 'primary_key' fails with removed value (value no longer gets dropped)
+* [fix] some index-settings not being ignored through `#transform_settings!`
+* [fix] `ActiveRecord::ConnectionAdapters::Elasticsearch::SchemaDumper` dumping environment-related tables in the same database
+* [fix] `ActiveRecord::ConnectionAdapters::Elasticsearch::TableMappingDefinition` fails with explicit assignable attributes (now uses ASSIGNABLE_ATTRIBUTES)
+* [fix] tables with provided 'table_name_prefix' or 'table_name_suffix' not being ignored by the SchemaDumper
+
+## [1.1.0] - 2022-12-01
 * [add] support for schema dumps & migrations for Elasticsearch
 * [add] `buckets` query/relation result method to resolve the buckets as key->value hash from aggregations
 * [add] support for third-party gems (e.g. elasticsearch-dsl)

data/lib/active_record/connection_adapters/elasticsearch/column.rb

@@ -5,13 +5,14 @@ module ActiveRecord
     module Elasticsearch
       class Column < ConnectionAdapters::Column # :nodoc:
 
-        attr_reader :virtual, :fields, :properties, :meta
+        attr_reader :virtual, :fields, :properties, :meta, :enabled
 
-        def initialize(name, default, sql_type_metadata = nil, virtual: false, fields: nil, properties: nil, meta: nil, **kwargs)
+        def initialize(name, default, sql_type_metadata = nil, virtual: false, fields: nil, properties: nil, meta: nil, enabled: nil, **kwargs)
           @virtual    = virtual
           @fields     = fields.presence || []
           @properties = properties.presence || []
           @meta       = meta.presence || {}
+          @enabled    = enabled.nil? ? true : enabled
 
           super(name, default, sql_type_metadata, true, nil, **kwargs)
         end
@@ -21,6 +22,12 @@ module ActiveRecord
           meta? && meta['comment']
         end
 
+        # returns true if this column is enabled (= searchable by queries)
+        # @return [Boolean]
+        def enabled?
+          !!enabled
+        end
+
         # returns true if this column is virtual.
         # Virtual columns cannot be saved.
         # @return [Boolean]

data/lib/active_record/connection_adapters/elasticsearch/schema_definitions/clone_table_definition.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'active_record/connection_adapters/elasticsearch/schema_definitions/table_definition'
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+      class CloneTableDefinition < TableDefinition
+
+        attr_reader :target
+
+        def initialize(conn, name, target, settings: nil, aliases: nil, **opts)
+          super(conn, name, **opts)
+
+          @target   = target
+          @settings = HashWithIndifferentAccess.new
+          @aliases  = HashWithIndifferentAccess.new
+
+          transform_settings!(settings) if settings.present?
+          transform_aliases!(aliases) if aliases.present?
+        end
+
+        # returns an array with all +TableSettingDefinition+.
+        # @return [Array]
+        def settings
+          @settings.values
+        end
+
+        # returns an array with all +TableAliasDefinition+.
+        # @return [Array]
+        def aliases
+          @aliases.values
+        end
+
+        ######################
+        # DEFINITION METHODS #
+        ######################
+
+        def setting(name, value, force: false, **options, &block)
+          raise ArgumentError, "you cannot define an already defined setting '#{name}'!" if @settings.key?(name) && !force?(force)
+
+          @settings[name] = new_setting_definition(name, value, **options, &block)
+
+          self
+        end
+
+        def remove_setting(name)
+          @settings.delete name
+        end
+
+        # we can use +alias+ here, since the instance method is not a reserved keyword!
+
+        def alias(name, force: false, **options, &block)
+          raise ArgumentError, "you cannot define an already defined alias '#{name}'." if @aliases.key?(name) && !force?(force)
+
+          @aliases[name] = new_alias_definition(name, **options, &block)
+
+          self
+        end
+
+        def remove_alias(name)
+          @aliases.delete name
+        end
+
+        private
+
+        def _before_exec
+          block_table(self.name, :write)
+        end
+
+        def _after_exec
+          unblock_table(self.name, :write)
+        end
+
+        alias :_rescue_exec :_after_exec
+
+        def _exec
+          execute(schema_creation.accept(self), 'CLONE TABLE').dig('acknowledged')
+        end
+
+        # force empty states to prevent "Name is static for an open table" error.
+        def state
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/elasticsearch/schema_definitions/column_methods.rb

@@ -17,7 +17,7 @@ module ActiveRecord
           # Appends a primary key definition to the table definition.
           # Can be called multiple times, but this is probably not a good idea.
           def primary_key(name, type = :primary_key, **options)
-            mapping(name, type, **options.merge(primary_key: true))
+            mapping(name, type, **options.merge(primary_key: true, auto_increment: true))
           end
         end
 

data/lib/active_record/connection_adapters/elasticsearch/schema_definitions/create_table_definition.rb

@@ -12,6 +12,7 @@ module ActiveRecord
           @settings = HashWithIndifferentAccess.new
           @mappings = HashWithIndifferentAccess.new
           @aliases  = HashWithIndifferentAccess.new
+          @metas    = HashWithIndifferentAccess.new
 
           transform_settings!(settings) if settings.present?
           transform_mappings!(mappings) if mappings.present?
@@ -30,6 +31,12 @@ module ActiveRecord
           @mappings.values
         end
 
+        # returns an array with all +TableMetaDefinition+.
+        # @return [Array]
+        def metas
+          @metas.values
+        end
+
         # provide backwards compatibility to columns
         alias columns mappings
 
@@ -48,11 +55,32 @@ module ActiveRecord
         # DEFINITION METHODS #
         ######################
 
+        # adds a new meta
+        def meta(name, value, force: false, **options)
+          raise ArgumentError, "you cannot define an already defined meta '#{name}'!" if @metas.key?(name) && !force?(force)
+
+          @metas[name] = new_meta_definition(name, value, **options)
+
+          self
+        end
+
+        def remove_meta(name)
+          @metas.delete(name)
+        end
+
         # adds a new mapping
         def mapping(name, type, force: false, **options, &block)
           raise ArgumentError, "you cannot define an already defined mapping '#{name}'!" if @mappings.key?(name) && !force?(force)
 
-          @mappings[name] = new_mapping_definition(name, type, **options, &block)
+          mapping = new_mapping_definition(name, type, **options, &block)
+          @mappings[name] = mapping
+
+
+          # check if the mapping is assigned as primary_key
+          if mapping.primary_key?
+            meta :primary_key, mapping.name
+            meta(:auto_increment, mapping.auto_increment) if mapping.auto_increment?
+          end
 
           self
         end
@@ -126,32 +154,6 @@ module ActiveRecord
         def state
           nil
         end
-
-        def transform_mappings!(mappings)
-          return unless mappings['properties'].present?
-
-          mappings['properties'].each do |name, attributes|
-            self.mapping(name, attributes.delete('type'), **attributes)
-          end
-        end
-
-        def transform_settings!(settings)
-          # exclude settings, that are provided through the API but are not part of the index-settings
-          settings
-            .with_indifferent_access
-            .each { |name, value|
-              # don't transform ignored names
-              next if ActiveRecord::ConnectionAdapters::Elasticsearch::TableSettingDefinition.match_ignore_names?(name)
-
-              self.setting(name, value)
-            }
-        end
-
-        def transform_aliases!(aliases)
-          aliases.each do |name, attributes|
-            self.alias(name, **attributes)
-          end
-        end
       end
     end
   end

data/lib/active_record/connection_adapters/elasticsearch/schema_definitions/table_definition.rb

@@ -10,16 +10,19 @@ module ActiveRecord
       class TableDefinition
         include ColumnMethods
 
-        delegate :execute, :schema_creation, :column_exists?, :mapping_exists?, :setting_exists?, :alias_exists?, :close_table, :open_table, :table_mappings, to: :conn
+        delegate :execute, :schema_creation,
+                 :column_exists?, :mapping_exists?, :meta_exists?, :setting_exists?, :alias_exists?,
+                 :close_table, :open_table, :block_table, :unblock_table,
+                 :table_mappings, :table_metas, :table_settings, to: :conn
 
         attr_reader :conn
         attr_reader :name
         attr_reader :opts
 
         def initialize(conn, name, **opts)
-          @conn = conn
-          @name = name
-          @opts = opts
+          @conn   = conn
+          @name   = name
+          @opts   = opts
           @failed = false
         end
 
@@ -33,10 +36,11 @@ module ActiveRecord
             yield self
           rescue => e
             @failed = false
+            _rescue_assign
             raise e
-          ensure
-            _after_assign
           end
+
+          _after_assign
         end
 
         def exec!
@@ -47,10 +51,11 @@ module ActiveRecord
             _exec
           rescue => e
             @failed = false
+            _rescue_exec
             raise e
-          ensure
-            _after_exec
           end
+
+          _after_exec
         end
 
         def failed?
@@ -67,6 +72,10 @@ module ActiveRecord
           true
         end
 
+        def _rescue_assign
+          true
+        end
+
         def _before_exec
           true
         end
@@ -75,11 +84,23 @@ module ActiveRecord
           true
         end
 
+        def _rescue_exec
+          true
+        end
+
         def _exec
           raise ArgumentError, "you cannot execute a TableDefinition directly - use 'CreateTableDefinition' or 'UpdateTableDefinition' instead!"
         end
 
-        def new_mapping_definition(name, type, strict: true, **attributes, &block)
+        def new_meta_definition(name, value, strict: false, **)
+          meta = TableMetaDefinition.new(name, value)
+
+          raise ArgumentError, "you cannot define an invalid meta '#{name}' (#{meta.error_messages})!" if strict?(strict) && !meta.valid?
+
+          meta
+        end
+
+        def new_mapping_definition(name, type, strict: false, **attributes, &block)
           mapping = TableMappingDefinition.new(name, type, attributes)
           block.call(mapping) if block_given?
 
@@ -90,7 +111,7 @@ module ActiveRecord
 
         alias :new_column_definition :new_mapping_definition
 
-        def new_alias_definition(name, strict: true, **attributes, &block)
+        def new_alias_definition(name, strict: false, **attributes, &block)
           # create new alias
           tbl_alias = TableAliasDefinition.new(name, attributes)
           block.call(tbl_alias) if block_given?
@@ -100,7 +121,7 @@ module ActiveRecord
           tbl_alias
         end
 
-        def new_setting_definition(name, value, strict: true, **, &block)
+        def new_setting_definition(name, value, strict: false, **, &block)
           # create new setting
           setting = TableSettingDefinition.new(name, value).with_state(state)
           block.call(setting) if block_given?
@@ -124,7 +145,41 @@ module ActiveRecord
         end
 
         def strict?(strict = nil)
-          opts.fetch(:strict, true) && strict != false
+          opts.fetch(:strict, false) || strict
+        end
+
+        def transform_mappings!(mappings)
+          # transform +_meta+ mappings
+          if mappings['_meta'].present?
+            mappings['_meta'].each do |name, value|
+              self.meta(name, value)
+            end
+          end
+
+          # transform properties (=columns)
+          if mappings['properties'].present?
+            mappings['properties'].each do |name, attributes|
+              self.mapping(name, attributes.delete('type'), **attributes)
+            end
+          end
+        end
+
+        def transform_settings!(settings)
+          # exclude settings, that are provided through the API but are not part of the index-settings
+          settings
+            .with_indifferent_access
+            .each { |name, value|
+              # don't transform ignored names
+              next if ActiveRecord::ConnectionAdapters::Elasticsearch::TableSettingDefinition.match_ignore_names?(name)
+
+              self.setting(name, value)
+            }
+        end
+
+        def transform_aliases!(aliases)
+          aliases.each do |name, attributes|
+            self.alias(name, **attributes)
+          end
         end
       end
     end

data/lib/active_record/connection_adapters/elasticsearch/schema_definitions/table_mapping_definition.rb

@@ -16,6 +16,9 @@ module ActiveRecord
                       :index_prefixes, :index, :meta, :normalizer, :norms, :null_value, :position_increment_gap,
                       :properties, :search_analyzer, :similarity, :subobjects, :store, :term_vector].freeze
 
+        # define virtual attributes, that must be assigned due a special logic
+        ASSIGNABLE_ATTRIBUTES = [:comment, :primary_key, :auto_increment, :meta].freeze
+
         build_attribute_methods! *ATTRIBUTES
 
         # attributes
@@ -27,10 +30,7 @@ module ActiveRecord
         validates_presence_of :name
         validates_presence_of :type
         validates_inclusion_of :__attributes_keys, in: ATTRIBUTES, allow_blank: true
-
-        # disable validation for meta attribute - maybe future updates of Elasticsearch have other restrictions.
-        # To not be hooked on those possible changes we
-        #validate :_validate_meta
+        validate :_validate_meta
 
         # sets the default value (alias for null_value)
         alias_method :default=, :null_value=
@@ -41,32 +41,62 @@ module ActiveRecord
         ####################
 
         def initialize(name, type, attributes)
-          @name       = name.to_sym
-          @attributes = attributes.symbolize_keys
+          @name = name.to_sym
+
+          attributes = attributes.symbolize_keys
+          # directly set attributes, that cannot be assigned
+          @attributes = attributes.except(*ASSIGNABLE_ATTRIBUTES)
+          # assign special attributes
+          __assign(attributes.slice(*ASSIGNABLE_ATTRIBUTES))
 
           @type = _resolve_type(type)
         end
 
-        # comment is handled as nested key from 'meta' attribute
+        # returns the +comment+ from 'meta' attribute
+        # @return [String, nil]
         def comment
           __get_nested(:meta, :comment)
         end
 
+        # sets the +comment+ as 'meta' attribute
+        # @param [String] value
         def comment=(value)
           # important: meta-values can only be strings!
           __set_nested(:meta, :comment, value.to_s)
         end
 
+        # returns true if the +primary_key+ 'attribute' was provided
+        # @return [Boolean]
         def primary_key
-          __get_nested(:meta, :primary_key) == 'true'
+          !!_lazy_attributes[:primary_key]
         end
 
         alias_method :primary_key?, :primary_key
 
-        # defines this mapping as a primary key
+        # sets the +primary_key+ as 'lazy_attribute'
+        # @param [Boolean] value
         def primary_key=(value)
-          # important: meta-values can only be strings!
-          __set_nested(:meta, :primary_key, value ? 'true' : nil)
+          _lazy_attributes[:primary_key] = value
+        end
+
+        # returns the +auto_increment+ value, if provided
+        # @return [Integer]
+        def auto_increment
+          return nil unless _lazy_attributes[:auto_increment]
+          return 0 if _lazy_attributes[:auto_increment] == true
+          _lazy_attributes[:auto_increment].to_i
+        end
+
+        # returns true if the +auto_increment+ 'attribute' was provided
+        # @return [Boolean]
+        def auto_increment?
+          !!auto_increment
+        end
+
+        # sets the +auto_increment+ as 'lazy_attribute'
+        # @param [Boolean, Integer] value
+        def auto_increment=(value)
+          _lazy_attributes[:auto_increment] = value
         end
 
         def meta=(value)
@@ -79,6 +109,11 @@ module ActiveRecord
 
         private
 
+        # non persistent attributes
+        def _lazy_attributes
+          @_lazy_attributes ||= {}
+        end
+
         # resolves the provided type.
         # prevents to set a nil type (sets +:object+ or +:nested+ - depends on existing properties)
         # @return [Symbol, nil]
@@ -99,8 +134,8 @@ module ActiveRecord
           return invalid!("'meta' is not supported on object or nested types", :attributes) if [:object, :nested].include?(type)
 
           # allow only strings
-          vkey = meta.keys.detect { |key| !meta[key].is_a?(String) }
-          return invalid!("'meta' has a key '#{vkey}' with a none string value", :attributes) if vkey.present?
+          failed_key = meta.keys.detect { |key| !meta[key].is_a?(String) }
+          return invalid!("'meta' has a key '#{failed_key}' with a none string value", :attributes) if failed_key.present?
 
           true
         end

data/lib/active_record/connection_adapters/elasticsearch/schema_definitions/table_meta_definition.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'active_model/validations'
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+      class TableMetaDefinition
+        include ActiveModel::Validations
+
+        # attributes
+        attr_accessor :name
+        attr_accessor :value
+
+        validates_presence_of :name
+
+        def initialize(name, value)
+          @name  = name.to_sym
+          @value = value
+        end
+      end
+    end
+  end
+end