elasticsearch_record 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aee5f9a1e1bc853f7e7648313db47e4b3179f85049baebc364cfce80a3e1dbdf
-  data.tar.gz: c3c17cd41fcc22930d28fdfdaedc69f5d5832d003e53141a0682942b8bb29496
+  metadata.gz: 6cf2af35e8ed5eb94892c5e1c6c5a7b11fab36539a48a3106b3444862c3654fc
+  data.tar.gz: 63f47637943c1e65e8d2a53c9f887b57ca5bdb139c4b6575997abbd640f2586d
 SHA512:
-  metadata.gz: e81eb876eb78e6133b4c8edc868f1d72fa2d3064d17680c6f0252f05d4c4029e3a6941dc3cd50948fc31ee91da308bb4f899c0744d708c5b4c66873a51985ba7
-  data.tar.gz: 5235d1a1e23ccce634d994fc037a6b15ec0471cbc0cb16e35891f61cd01bad39cedc16c390a569b958677dada149825cdb514d59bb6d85465182b583974deeb7
+  metadata.gz: 96f3ce41b50dc97f5e4d9cdf79bf5b58af5bedcf219edee5d5d22a88b6517efcad0f3d7a0e26ca0042493d83a59d274a3f33d80ec4e16032c7ec50dfcb6790b2
+  data.tar.gz: c041e7d0bc2e00dd006d1c5b87e403b03f2e9c98a55672f636b29821a318425dc72aabff46e439a4c1281c914f1a07ac86487961f06930fac4169d538a2b9888

data/.yardopts

@@ -0,0 +1,4 @@
+--embed-mixins
+--markup=markdown
+--files docs/*.*
+--plugin relative_markdown_links

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    elasticsearch_record (0.1.0)
+    elasticsearch_record (1.1.0)
       activerecord (~> 7.0.0)
       elasticsearch (~> 8.4)
 
@@ -20,19 +20,18 @@ GEM
       tzinfo (~> 2.0)
     concurrent-ruby (1.1.10)
     diff-lcs (1.5.0)
-    docile (1.4.0)
     elastic-transport (8.1.0)
       faraday (< 3)
       multi_json
-    elasticsearch (8.4.0)
+    elasticsearch (8.5.2)
       elastic-transport (~> 8)
-      elasticsearch-api (= 8.4.0)
-    elasticsearch-api (8.4.0)
+      elasticsearch-api (= 8.5.2)
+    elasticsearch-api (8.5.2)
       multi_json
-    faraday (2.5.2)
+    faraday (2.7.1)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
-    faraday-net_http (3.0.0)
+    faraday-net_http (3.0.2)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     minitest (5.16.3)
@@ -52,14 +51,11 @@ GEM
       rspec-support (~> 3.11.0)
     rspec-support (3.11.1)
     ruby2_keywords (0.0.5)
-    simplecov (0.21.2)
-      docile (~> 1.1)
-      simplecov-html (~> 0.11)
-      simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.4)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
+    webrick (1.7.0)
+    yard (0.9.28)
+      webrick (~> 1.7.0)
 
 PLATFORMS
   x86_64-linux
@@ -68,7 +64,7 @@ DEPENDENCIES
   elasticsearch_record!
   rake (~> 13.0)
   rspec (~> 3.0)
-  simplecov (~> 0.21)
+  yard (~> 0.9)
 
 BUNDLED WITH
    2.3.18

data/README.md

@@ -1,6 +1,10 @@
 # ElasticsearchRecord
 
+[![GitHub](https://img.shields.io/badge/github-ruby--smart/elasticsearch_record-blue.svg)](http://github.com/ruby-smart/elasticsearch_record)
+[![Documentation](https://img.shields.io/badge/docs-rdoc.info-blue.svg)](http://rubydoc.info/gems/elasticsearch_record)
+
 [![Gem Version](https://badge.fury.io/rb/elasticsearch_record.svg)](https://badge.fury.io/rb/elasticsearch_record)
+[![License](https://img.shields.io/github/license/ruby-smart/elasticsearch_record)](docs/LICENSE.txt)
 
 ActiveRecord adapter for Elasticsearch
 
@@ -9,6 +13,7 @@ _ElasticsearchRecord is a ActiveRecord adapter and provides similar functionalit
 -----
 
 **PLEASE NOTE:**
+
 - This is still in **development**!
 - Specs & documentation will follow. 
 - You might experience BUGs and Exceptions...
@@ -34,19 +39,25 @@ Or install it yourself as:
 
 
 ## Features
-* CRUD: Reading and Writing Data as already used for ActiveRecord models ```create, update, delete```
-* Query-chaining through the Active Record Query Interface
-* Query interface with additional methods for ```filter, must, must_not, should```
-* Aggregation queries ```aggregate```
+* ActiveRecord's `create, read, update & delete` behaviours
+* Active Record Query Interface
+  * query-chaining
+  * scopes
+  * additional relation methods to find records with `filter, must, must_not, should`
+  * aggregated queries with Elasticsearch `aggregation` methods
+  * resolve search response `hits`, `aggregations`, `buckets`, ... instead of ActiveRecord objects
+* Third-party gem support
+  * access `elasticsearch-dsl` query builder through `model.search{ ... }`
+* Schema
+  * dump
+  * create & update of tables _(indices)_ with mappings, settings & aliases
 * Instrumentation for ElasticsearchRecord
+  * logs Elasticsearch API-calls
+  * shows Runtime in logs
 
 ## Contra - what it _(currently)_ can not
-* Query-based associations like 'has_one' through a single _(or multiple)_ queries - aka. joins
+* Joins to other indexes or databases
 * complex, combined or nested queries ```and, or, Model.arel ...```
-* Create mappings / schema through migrations _(so no create_table, update_column, ...)_
-* Schema dumps
-* Manage indexes and mappings
-
 
 ## Setup
 
@@ -103,11 +114,19 @@ end
 
 ### d) have FUN with your model:
 ```ruby
-scope = Search.filter(term: {name: 'MyImportantObject'}).limit(5)
+scope = Search
+        .where(name: 'Custom Object Name')
+        .where(token: nil)
+        .filter(terms: {type: [:x, :y]})
+        .limit(5)
+
+# take the first object
 obj = scope.take
 
+# update the objects name
 obj.update(name: "Not-So-Important")
 
+# extend scope and update all docs
 scope.where(kind: :undefined).offset(10).update_all(name: "New Name")
 
 ```
@@ -115,7 +134,6 @@ scope.where(kind: :undefined).offset(10).update_all(name: "New Name")
 ## Active Record Query Interface
 
 ### Refactored ```where``` method:
-
 Different to the default where-method you can now use it in different ways.
 
 Using it by default with a Hash, the method decides itself to either add a filter, or must_not query.
@@ -141,17 +159,42 @@ Search.where(:should, term: {name: 'Mano'})
 # > should: {term: {name: 'Mano'}}
 ```
 
-### Usage Examples
+### Result methods:
+You can simply return RAW data without instantiating ActiveRecord objects:
+
 ```ruby
-# save a new record
-model = Search.new(name: "Cool object", kind: "open")
-model.save
 
-# find a record by id
-MyEsIndex.find_by_id("xyzAbc34")
+# returns the response RAW hits hash.
+hits = Search.where(name: 'A nice object').hits
+# > {"total"=>{"value"=>5, "relation"=>"eq"}, "max_score"=>1.0, "hits"=>[{ "_index": "search", "_type": "_doc", "_id": "abc123", "_score": 1.0, "_source": { "name": "A nice object", ...
+
+# Returns the RAW +_source+ data from each hit - aka. +rows+.
+results = Search.where(name: 'A nice object').results
+# > [{ "name": "A nice object", ...
+
+# returns the response RAW aggregations hash.
+aggs = Search.where(name: 'A nice object').aggregate(:total, {sum: {field: :amount}}).aggregations
+# > {"total"=>{"value"=>6722604.0}}
+
+# returns the (nested) bucket values (and aggregated values) from the response aggregations.
+buckets = Search.where(name: 'A nice object').aggregate(:total, {sum: {field: :amount}}).buckets
+# > {"total"=>6722604.0}
+
+# resolves RAW +_source+ data from each hit with a +point_in_time+ query (also includes _id)
+# useful if you want more then 10000 results.
+results = Search.where(name: 'A nice object').pit_results
+# > [{ "_id": "abc123", "name": "A nice object", ...
+
+# returns the total value of the query without querying again (it uses the total value from the response)
+scope = Search.where(name: 'A nice object').limit(5)
+results_count = scope.count
+# > 5
+total = scope.total
+# > 3335
+
 ```
 
-### Useful chain methods
+### Available query/relation chain methods
 - kind 
 - configure
 - aggregate
@@ -161,8 +204,12 @@ MyEsIndex.find_by_id("xyzAbc34")
 - must
 - should
 - aggregate
+- restrict 
+- hits_only!
+- aggs_only!
+- total_only!
 
-### Useful calculation methods
+### Available calculation methods
 - percentiles
 - percentile_ranks
 - cardinality
@@ -172,8 +219,9 @@ MyEsIndex.find_by_id("xyzAbc34")
 - sum
 - calculate
 
-### Useful result methods
+### Available result methods
 - aggregations
+- buckets
 - hits
 - results
 - total
@@ -188,31 +236,136 @@ MyEsIndex.find_by_id("xyzAbc34")
 
 -----
 
-### Useful model-class attributes
+### Useful model class attributes
 - index_base_name
+- relay_id_attribute
 
-### Useful model methods
+### Useful model instance methods
 - source_column_names
 - searchable_column_names
 - find_by_query
 - msearch
 
+## ActiveRecord ConnectionAdapters schema-methods
+Access these methods through the model's connection.
+```ruby
+  # returns mapping of provided table (index)
+  model.connection.table_mappings('table-name')
+```
+
+- table_mappings
+- table_settings
+- table_aliases
+- table_state
+- table_schema
+- alias_exists?
+- setting_exists?
+- mapping_exists?
+- max_result_window
+- cluster_info
+
+## Active Record Schema migration methods
+Access these methods through the model's connection or within any `Migration`.
+
+- open_table
+- open_tables
+- close_table
+- close_tables
+- truncate_table
+- truncate_tables
+- drop_table
+- create_table
+- change_table
+- add_mapping
+- change_mapping
+- change_mapping_meta
+- change_mapping_attributes
+- add_setting
+- change_setting
+- delete_setting
+- add_alias
+- change_alias
+- delete_alias
+
+```ruby
+# Example migration
+class AddTests < ActiveRecord::Migration[7.0]
+  def up
+    create_table "assignments", if_not_exists: true do |t|
+      t.string :key, primary_key: true
+      t.text :value
+      t.timestamps
+
+      t.setting :number_of_shards, "1"
+      t.setting :number_of_replicas, 0
+    end
+    
+    create_table "settings", force: true do |t|
+      t.mapping :created_at, :date
+      t.mapping :key, :keyword do |m|
+        m.primary_key = true
+      end
+      t.mapping :status, :keyword
+      t.mapping :updated_at, :date
+      t.mapping :value, :text
+
+      t.setting "index.number_of_replicas", "0"
+      t.setting "index.number_of_shards", "1"
+      t.setting "index.routing.allocation.include._tier_preference", "data_content"
+    end
+
+    add_mapping "settings", :active, :boolean do |m|
+      m.comment = "Hans"
+    end
+
+    change_table 'settings', force: true do |t|
+      t.add_setting( "index.search.idle.after", "20s")
+      t.add_setting("index.shard.check_on_startup", true)
+      t.add_alias('supersettings')
+    end
+
+    delete_alias('settings', :supersettings)
+    delete_setting('settings', 'index.search.idle.after')
+
+    change_table 'settings', force: true do |t|
+      t.integer :amount_of_newbies
+    end
+    
+    create_table "vintage", force: true do |t|
+      t.primary_key :number
+      t.string :name
+      t.timestamps
+    end
+
+    change_mapping_meta "vintage", :number, auto_increment: 'true'
+    change_mapping_meta "vintage", :number, peter: 'hans'
+
+    change_mapping_attributes "vintage", :number, fields: {raw: {type: :keyword}}
+  end
+
+  def down
+    drop_table 'assignments'
+    drop_table 'settings'
+    drop_table 'vintage'
+  end
+end
+```
 
 ## Docs
 
-[CHANGELOG](./docs/CHANGELOG.md)
+[CHANGELOG](docs/CHANGELOG.md)
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at [elasticsearch_record](https://github.com/ruby-smart/elasticsearch_record).
-This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](./docs/CODE_OF_CONDUCT.md).
+Bug reports and pull requests are welcome on [GitHub](https://github.com/ruby-smart/elasticsearch_record).
+This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](docs/CODE_OF_CONDUCT.md).
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
 
-A copy of the [LICENSE](./docs/LICENSE.txt) can be found @ the docs.
+A copy of the [LICENSE](docs/LICENSE.txt) can be found @ the docs.
 
 ## Code of Conduct
 
-Everyone interacting in the project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [CODE OF CONDUCT](./docs/CODE_OF_CONDUCT.md).
+Everyone interacting in the project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [CODE OF CONDUCT](docs/CODE_OF_CONDUCT.md).

data/docs/CHANGELOG.md

@@ -1,43 +1,61 @@
 # ElasticsearchRecord - CHANGELOG
 
+## [1.1.0] - 2022-12-02
+* [add] support for schema dumps & migrations for Elasticsearch
+* [add] `buckets` query/relation result method to resolve the buckets as key->value hash from aggregations
+* [add] support for third-party gems (e.g. elasticsearch-dsl)
+* [add] custom primary_key support with 'auto_increment' adaption _(beta)_
+* [ref] instrumentation of `LogSubscriber` coloring
+* [ref] unsupported methods to show exception
+* [ref] query/relation method `#msearch` to support options 'resolve', 'transpose' & 'keep_null_relation'
+* [ref] `ActiveRecord::ConnectionAdapters::ElasticsearchAdapter#translate_exception` for better error handling
+* [ref] `ActiveRecord::ConnectionAdapters::Elasticsearch::Column` for a much easier assignment
+* [ref] gemspec, yardoc & docs
+* [fix] `ActiveRecord::ConnectionAdapters::ElasticsearchAdapter::BASE_STRUCTURE` to force primary_key column '_id' as virtual
+* [fix] `ElasticsearchRecord::Relation::QueryClauseTree` not correctly calculating & merging 'filters, musts, ... '
+* [fix] failing query not really failing in nested queries (msearch) 
+* [fix] 'null_realation' being executed in msearch
+* [fix] quoting of columns, attributes & values
+* [fix] query/relation method `#count` not preventing 'null_relation?'
+
 ## [1.0.2] - 2022-11-07
-* [add] ```ActiveRecord::ConnectionAdapters::ElasticsearchAdapter#migrations_paths``` with 'db/migrate_elasticsearch'
+* [add] `ActiveRecord::ConnectionAdapters::ElasticsearchAdapter#migrations_paths` with 'db/migrate_elasticsearch'
 * [fix] to prevent executing 'primary' migrations to elasticsearch (SchemaDumper may still throw error comments)
 * [add] temporary workarounds for scheme & migrations until we support it (so bin/rails db:migrate - tasks will now run again)
 
 ## [1.0.1] - 2022-10-19
-* [add] ```ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Nested``` class to cast nested values
+* [add] `ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Nested` class to cast nested values
 * [add] **properties** to column definition (so they are also searchable by _Relation_ conditions)
 * [add] exception for _Relation_ #pit_results if batch size is too large
 * [add] a default _#find_by_id_-method to proved a 'fallback' functionality for the primary '_id' column
 * [fix] nested properties are not cast for column-type "object"
-* [fix] ```ActiveRecord::ConnectionAdapters::Elasticsearch::SchemaStatements``` fields and property detection
-* [fix] ```ElasticsearchRecord::StatementCache::PartialQuery``` reference manipulation of cached hash _(missing .deep_dup )_
-* [ref] ```ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Object``` class to only cast object values
-* [ref] ```Arel::Visitors::Elasticsearch#visit_Sort``` to detect a random sort with correct keyword: "**\_\_rand\_\_**"
+* [fix] `ActiveRecord::ConnectionAdapters::Elasticsearch::SchemaStatements` fields and property detection
+* [fix] `ElasticsearchRecord::StatementCache::PartialQuery` reference manipulation of cached hash _(missing .deep_dup )_
+* [ref] `ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Object` class to only cast object values
+* [ref] `Arel::Visitors::Elasticsearch#visit_Sort` to detect a random sort with correct keyword: "**\_\_rand\_\_**"
 
 ## [1.0.0] - 2022-10-18
-* [add] patch for ```ActiveRecord::Relation::Merger``` - to support AR-relations
-* [add] ```ElasticsearchRecord::Relation#pit_results``` _offset_ & _yield_ support
-* [add] individual instrumentation names for ```ElasticsearchRecord::Relation``` result methods (like: #aggregations)
+* [add] patch for `ActiveRecord::Relation::Merger` - to support AR-relations
+* [add] `ElasticsearchRecord::Relation#pit_results` _offset_ & _yield_ support
+* [add] individual instrumentation names for `ElasticsearchRecord::Relation` result methods (like: #aggregations)
 * [rem] cleanup Debugging & logging
 * [fix] quoting for any values
-* [fix] calculation ```count``` method to support already known syntax (with column, distinct, limited, ...)
-* [fix] ```Arel::Nodes``` to support additional args (query, kind, aggs, ...)
+* [fix] calculation `count` method to support already known syntax (with column, distinct, limited, ...)
+* [fix] `Arel::Nodes` to support additional args (query, kind, aggs, ...)
 * [fix] _relation manager patch_ to not mash up different relations
-* [fix] ```ElasticsearchRecord``` total calculation for failed queries
-* [fix] ```ElasticsearchRecord::Relation#count``` to correctly _unscope_
-* [fix] ```Arel::Visitors::Elasticsearch```
+* [fix] `ElasticsearchRecord` total calculation for failed queries
+* [fix] `ElasticsearchRecord::Relation#count` to correctly _unscope_
+* [fix] `Arel::Visitors::Elasticsearch`
   * build query where-clauses without existing default query-"kind" 
   * directly fail if a grouping _(visit_Arel_Nodes_Grouping)_ was provided
   * forced _failed!_ state was not correctly claimed
   * buggy _assign_-method for nested arrays
-* [fix] ```Arel::Collectors::ElasticsearchQuery```
+* [fix] `Arel::Collectors::ElasticsearchQuery`
   * claim for _argument_
   * delete key on _nil_ assign
-* [ref] simplify ```Arel::Collectors::ElasticsearchQuery``` (remove stack & scoping)
-* [ref] simplify ```Arel::Visitors::Elasticsearch``` to support binds (statement cache) and simple where predicates
-* [ref] rename ```ElasticsearchRecord::Query#arguments``` -> _#query_arguments_
+* [ref] simplify `Arel::Collectors::ElasticsearchQuery` (remove stack & scoping)
+* [ref] simplify `Arel::Visitors::Elasticsearch` to support binds (statement cache) and simple where predicates
+* [ref] rename `ElasticsearchRecord::Query#arguments` -> _#query_arguments_
 
 ## [0.1.2] - 2022-09-23
 * [fix] Records / Elasticsearch index with additional 'id' fields not recognizing

data/docs/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2022 Tobias Gonsior
+Copyright (c) 2022 Ruby Smart
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/elasticsearch_record.gemspec

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "lib/elasticsearch_record/version"
+
+Gem::Specification.new do |spec|
+  spec.name        = "elasticsearch_record"
+  spec.version     = ElasticsearchRecord.version
+  spec.authors     = ["Tobias Gonsior"]
+  spec.email       = ["info@ruby-smart.org"]
+  spec.summary     = "ActiveRecord adapter for Elasticsearch"
+  spec.description = <<DESC
+ElasticsearchRecord is a ActiveRecord adapter and provides similar functionality for Elasticsearch.
+DESC
+
+  spec.homepage              = "https://github.com/ruby-smart/elasticsearch_record"
+  spec.license               = "MIT"
+  spec.required_ruby_version = ">= 2.7.0"
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+  spec.metadata["homepage_uri"]      = spec.homepage
+  spec.metadata["source_code_uri"]   = "https://github.com/ruby-smart/elasticsearch_record"
+  spec.metadata['documentation_uri'] = 'https://rubydoc.info/gems/elasticsearch_record'
+  spec.metadata["changelog_uri"]     = "#{spec.metadata["source_code_uri"]}/blob/main/docs/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'activerecord', '~> 7.0.0'
+  spec.add_dependency 'elasticsearch', '~> 8.4'
+
+  #spec.add_development_dependency 'coveralls_reborn', '~> 0.25'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rake', "~> 13.0"
+  spec.add_development_dependency 'yard', '~> 0.9'
+end

data/lib/active_record/connection_adapters/elasticsearch/column.rb

@@ -5,13 +5,20 @@ module ActiveRecord
     module Elasticsearch
       class Column < ConnectionAdapters::Column # :nodoc:
 
-        attr_reader :virtual, :fields, :properties
+        attr_reader :virtual, :fields, :properties, :meta
 
-        def initialize(name, default, sql_type_metadata = nil, null = true, default_function = nil, **kwargs)
-          @virtual    = kwargs.delete(:virtual)
-          @fields     = kwargs.delete(:fields)
-          @properties = kwargs.delete(:properties)
-          super(name, default, sql_type_metadata, null, default_function, **kwargs)
+        def initialize(name, default, sql_type_metadata = nil, virtual: false, fields: nil, properties: nil, meta: nil, **kwargs)
+          @virtual    = virtual
+          @fields     = fields.presence || []
+          @properties = properties.presence || []
+          @meta       = meta.presence || {}
+
+          super(name, default, sql_type_metadata, true, nil, **kwargs)
+        end
+
+        # returns comment from meta
+        def comment
+          meta? && meta['comment']
         end
 
         # returns true if this column is virtual.
@@ -21,6 +28,13 @@ module ActiveRecord
           !!virtual
         end
 
+        # returns true if this column has meta information
+        # To receive the nested meta-data just call +#meta+ on this object.
+        # @return [Boolean]
+        def meta?
+          meta.present?
+        end
+
         # returns true if this column has nested fields
         # To receive the nested names just call +#fields+ on this object.
         # @return [Boolean]