search_flip 3.8.0 → 4.0.0.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +12 -38
- data/.rubocop.yml +3 -7
- data/CHANGELOG.md +6 -43
- data/Gemfile +1 -15
- data/README.md +39 -53
- data/UPDATING.md +40 -0
- data/docker-compose.yml +0 -1
- data/lib/search_flip/aggregation.rb +1 -1
- data/lib/search_flip/aws_sigv4_plugin.rb +1 -0
- data/lib/search_flip/bulk.rb +2 -2
- data/lib/search_flip/config.rb +1 -7
- data/lib/search_flip/connection.rb +23 -91
- data/lib/search_flip/criteria.rb +10 -37
- data/lib/search_flip/filterable.rb +0 -16
- data/lib/search_flip/http_client.rb +7 -46
- data/lib/search_flip/index.rb +10 -16
- data/lib/search_flip/json.rb +3 -3
- data/lib/search_flip/response.rb +7 -8
- data/lib/search_flip/result.rb +45 -19
- data/lib/search_flip/to_json.rb +29 -1
- data/lib/search_flip/version.rb +1 -1
- data/lib/search_flip.rb +3 -10
- data/search_flip.gemspec +12 -6
- data/spec/search_flip/aggregation_spec.rb +17 -17
- data/spec/search_flip/aws_sigv4_plugin_spec.rb +6 -10
- data/spec/search_flip/bulk_spec.rb +8 -5
- data/spec/search_flip/connection_spec.rb +7 -110
- data/spec/search_flip/criteria_spec.rb +13 -60
- data/spec/search_flip/http_client_spec.rb +3 -13
- data/spec/search_flip/index_spec.rb +13 -34
- data/spec/search_flip/json_spec.rb +4 -18
- data/spec/search_flip/null_instrumenter_spec.rb +2 -2
- data/spec/search_flip/response_spec.rb +2 -2
- data/spec/search_flip/result_spec.rb +23 -6
- data/spec/search_flip/to_json_spec.rb +28 -0
- data/spec/spec_helper.rb +4 -6
- metadata +142 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b023bc6411647eaecfcb8f5b0fe535379974a0824d49699dd74ef867a67c69eb
|
4
|
+
data.tar.gz: 5dc4a0439f9442811f3966e86b0c6b09d09c54450a04234c3738ae0ab7935900
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a5fc637e8c016531df22af678adcae60a4f4169e5f1d2215d52be0a0dff1111b849cb4d79ca36d6cbf315c8516bc51d543852ab5026339d59cb1f5d08ba356f7
|
7
|
+
data.tar.gz: 7cf9d55a9aca5d6d63401750d40bc656fcc345101d8ce92a300c24649e38810d399d43ff225611bbdd91a257d6cb15eee26a6c811974b987e92e72e67eded704
|
data/.github/workflows/test.yml
CHANGED
@@ -7,54 +7,28 @@ jobs:
|
|
7
7
|
fail-fast: false
|
8
8
|
matrix:
|
9
9
|
elasticsearch:
|
10
|
-
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
-
|
15
|
-
env:
|
16
|
-
discovery.type: single-node
|
17
|
-
xpack.security.enabled: false
|
18
|
-
- image: docker.elastic.co/elasticsearch/elasticsearch:6.7.0
|
19
|
-
env:
|
20
|
-
discovery.type: single-node
|
21
|
-
xpack.security.enabled: false
|
22
|
-
- image: docker.elastic.co/elasticsearch/elasticsearch:7.0.0
|
23
|
-
env:
|
24
|
-
discovery.type: single-node
|
25
|
-
xpack.security.enabled: false
|
26
|
-
- image: docker.elastic.co/elasticsearch/elasticsearch:7.11.2
|
27
|
-
env:
|
28
|
-
discovery.type: single-node
|
29
|
-
xpack.security.enabled: false
|
30
|
-
- image: docker.elastic.co/elasticsearch/elasticsearch:8.1.1
|
31
|
-
env:
|
32
|
-
discovery.type: single-node
|
33
|
-
xpack.security.enabled: false
|
34
|
-
- image: opensearchproject/opensearch:1.3.14
|
35
|
-
env:
|
36
|
-
discovery.type: single-node
|
37
|
-
plugins.security.disabled: true
|
38
|
-
- image: opensearchproject/opensearch:2.11.1
|
39
|
-
env:
|
40
|
-
discovery.type: single-node
|
41
|
-
plugins.security.disabled: true
|
10
|
+
- plainpicture/elasticsearch:2.4.1_delete-by-query
|
11
|
+
- elasticsearch:5.4
|
12
|
+
- docker.elastic.co/elasticsearch/elasticsearch:6.7.0
|
13
|
+
- docker.elastic.co/elasticsearch/elasticsearch:7.0.0
|
14
|
+
- docker.elastic.co/elasticsearch/elasticsearch:7.9.0
|
42
15
|
ruby:
|
16
|
+
- 2.5
|
17
|
+
- 2.6
|
43
18
|
- 2.7
|
44
|
-
- 3.0
|
45
|
-
- 3.1
|
46
|
-
- 3.2
|
47
19
|
services:
|
48
20
|
elasticsearch:
|
49
|
-
image: ${{ matrix.elasticsearch
|
50
|
-
env:
|
21
|
+
image: ${{ matrix.elasticsearch }}
|
22
|
+
env:
|
23
|
+
discovery.type: single-node
|
51
24
|
ports:
|
52
25
|
- 9200:9200
|
53
26
|
steps:
|
54
27
|
- uses: actions/checkout@v1
|
55
|
-
- uses:
|
28
|
+
- uses: actions/setup-ruby@v1
|
56
29
|
with:
|
57
30
|
ruby-version: ${{ matrix.ruby }}
|
58
31
|
- run: bundle
|
32
|
+
- run: sleep 10
|
59
33
|
- run: bundle exec rspec
|
60
34
|
- run: bundle exec rubocop
|
data/.rubocop.yml
CHANGED
@@ -1,15 +1,11 @@
|
|
1
1
|
AllCops:
|
2
2
|
NewCops: enable
|
3
|
-
TargetRubyVersion: 2.
|
4
|
-
SuggestExtensions: false
|
3
|
+
TargetRubyVersion: 2.4
|
5
4
|
|
6
|
-
Gemspec/
|
5
|
+
Gemspec/RequiredRubyVersion:
|
7
6
|
Enabled: false
|
8
7
|
|
9
|
-
|
10
|
-
EmptyLineBetweenClassDefs: false
|
11
|
-
|
12
|
-
Gemspec/RequiredRubyVersion:
|
8
|
+
Style/CaseLikeIf:
|
13
9
|
Enabled: false
|
14
10
|
|
15
11
|
Style/ExplicitBlockArgument:
|
data/CHANGELOG.md
CHANGED
@@ -1,50 +1,13 @@
|
|
1
1
|
|
2
2
|
# CHANGELOG
|
3
3
|
|
4
|
-
##
|
4
|
+
## v4.0.0
|
5
5
|
|
6
|
-
*
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
*
|
11
|
-
|
12
|
-
## v3.7.1
|
13
|
-
|
14
|
-
* Fix thread-safety issue of http-rb
|
15
|
-
|
16
|
-
## v3.7.0
|
17
|
-
|
18
|
-
* Add `SearchFlip::Connection#bulk` to allow more clean bulk indexing to
|
19
|
-
multiple indices at once
|
20
|
-
|
21
|
-
## v3.6.0
|
22
|
-
|
23
|
-
* Support Elasticsearch v8
|
24
|
-
|
25
|
-
## v3.5.0
|
26
|
-
|
27
|
-
* Add `SearchFlip::Criteria#http_timeout` to allow specifying timeouts on
|
28
|
-
a query level
|
29
|
-
|
30
|
-
## v3.4.0
|
31
|
-
|
32
|
-
* Expose `Http#timeout` via `SearchFlip::HTTPClient`
|
33
|
-
|
34
|
-
## v3.3.0
|
35
|
-
|
36
|
-
* Update httprb
|
37
|
-
* Changed oj default options
|
38
|
-
* Allow to set oj json options
|
39
|
-
|
40
|
-
## v3.2.1
|
41
|
-
|
42
|
-
* Fix `refresh` having a empty body breaking in elasticsearch 7.11
|
43
|
-
|
44
|
-
## v3.2.0
|
45
|
-
|
46
|
-
* Fix `index_scope` not being passed in `each_record` without block
|
47
|
-
* Added `SearchFlip::Criteria#match_none`
|
6
|
+
* [BREAKING] For performance reasons, `SearchFlip::Result` now no longer
|
7
|
+
inherits `Hashie::Mash`
|
8
|
+
* It no longer supports symbol based access like `result[:id]`
|
9
|
+
* It no longer supports question mark methods like `result.title?`
|
10
|
+
* It no longer supports method based assignment like `result.some_key = "value"`
|
48
11
|
|
49
12
|
## v3.1.2
|
50
13
|
|
data/Gemfile
CHANGED
@@ -2,18 +2,4 @@ source "https://rubygems.org"
|
|
2
2
|
|
3
3
|
gemspec
|
4
4
|
|
5
|
-
if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.2.2")
|
6
|
-
gem "activerecord", ">= 3.0", "< 5"
|
7
|
-
else
|
8
|
-
gem "activerecord", ">= 3.0"
|
9
|
-
end
|
10
|
-
|
11
|
-
gem "aws-sdk-core"
|
12
|
-
gem "bundler"
|
13
|
-
gem "factory_bot"
|
14
|
-
gem "rake"
|
15
|
-
gem "rspec"
|
16
|
-
gem "rubocop"
|
17
|
-
gem "sqlite3"
|
18
|
-
gem "timecop"
|
19
|
-
gem "webmock"
|
5
|
+
gem "activerecord", "< 5" if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.2.2")
|
data/README.md
CHANGED
@@ -9,9 +9,8 @@
|
|
9
9
|
Using SearchFlip it is dead-simple to create index classes that correspond to
|
10
10
|
[Elasticsearch](https://www.elastic.co/) indices and to manipulate, query and
|
11
11
|
aggregate these indices using a chainable, concise, yet powerful DSL. Finally,
|
12
|
-
SearchFlip supports Elasticsearch 2.x, 5.x, 6.x, 7.x
|
13
|
-
|
14
|
-
version dependent features.
|
12
|
+
SearchFlip supports Elasticsearch 2.x, 5.x, 6.x, 7.x. Check section
|
13
|
+
[Feature Support](#feature-support) for version dependent features.
|
15
14
|
|
16
15
|
```ruby
|
17
16
|
CommentIndex.search("hello world", default_field: "title").where(visible: true).aggregate(:user_id).sort(id: "desc")
|
@@ -52,7 +51,7 @@ CommentIndex.search("hello world").where(available: true).sort(id: "desc").aggre
|
|
52
51
|
|
53
52
|
```
|
54
53
|
|
55
|
-
Finally, SearchFlip comes with a minimal set of dependencies.
|
54
|
+
Finally, SearchFlip comes with a minimal set of dependencies (http-rb and oj only).
|
56
55
|
|
57
56
|
## Reference Docs
|
58
57
|
|
@@ -419,14 +418,6 @@ Simply matches all documents:
|
|
419
418
|
CommentIndex.match_all
|
420
419
|
```
|
421
420
|
|
422
|
-
* `match_none`
|
423
|
-
|
424
|
-
Simply matches none documents at all:
|
425
|
-
|
426
|
-
```ruby
|
427
|
-
CommentIndex.match_none
|
428
|
-
```
|
429
|
-
|
430
421
|
* `all`
|
431
422
|
|
432
423
|
Simply returns the criteria as is or an empty criteria when called on the index
|
@@ -483,8 +474,8 @@ end
|
|
483
474
|
```
|
484
475
|
|
485
476
|
Generally, aggregation results returned by Elasticsearch are returned as a
|
486
|
-
`SearchFlip::Result`, which basically is a
|
487
|
-
access them via:
|
477
|
+
`SearchFlip::Result`, which basically is a Hash with method-like access, such
|
478
|
+
that you can access them via:
|
488
479
|
|
489
480
|
```ruby
|
490
481
|
query.aggregations(:username)["mrkamel"].revenue.value
|
@@ -699,14 +690,6 @@ Specify a timeout to limit query processing time:
|
|
699
690
|
CommentIndex.timeout("3s").execute
|
700
691
|
```
|
701
692
|
|
702
|
-
* `http_timeout`
|
703
|
-
|
704
|
-
Specify a http timeout for the request which will be send to Elasticsearch:
|
705
|
-
|
706
|
-
```ruby
|
707
|
-
CommentIndex.http_timeout(3).execute
|
708
|
-
```
|
709
|
-
|
710
693
|
* `terminate_after`
|
711
694
|
|
712
695
|
Activate early query termination to stop query processing after the specified
|
@@ -765,7 +748,7 @@ end
|
|
765
748
|
This allows to use different clusters per index e.g. when migrating indices to
|
766
749
|
new versions of Elasticsearch.
|
767
750
|
|
768
|
-
You can specify basic auth, additional headers,
|
751
|
+
You can specify basic auth, additional headers, etc via:
|
769
752
|
|
770
753
|
```ruby
|
771
754
|
http_client = SearchFlip::HTTPClient.new
|
@@ -782,9 +765,6 @@ http_client = http_client.via("proxy.host", 8080)
|
|
782
765
|
# Custom headers
|
783
766
|
http_client = http_client.headers(key: "value")
|
784
767
|
|
785
|
-
# Timeouts
|
786
|
-
http_client = http_client.timeout(20)
|
787
|
-
|
788
768
|
SearchFlip::Connection.new(base_url: "...", http_client: http_client)
|
789
769
|
```
|
790
770
|
|
@@ -853,7 +833,7 @@ performance tracing, etc.
|
|
853
833
|
To use instrumentation, configure the instrumenter:
|
854
834
|
|
855
835
|
```ruby
|
856
|
-
SearchFlip::Config[:instrumenter] = ActiveSupport::Notifications
|
836
|
+
SearchFlip::Config[:instrumenter] = ActiveSupport::Notifications.notifier
|
857
837
|
```
|
858
838
|
|
859
839
|
Subsequently, you can subscribe to notifcations for `request.search_flip`:
|
@@ -894,52 +874,57 @@ Thus, if your ORM supports `.find_each`, `#id` and `#where` you are already
|
|
894
874
|
good to go. Otherwise, simply add your custom implementation of those methods
|
895
875
|
that work with whatever ORM you use.
|
896
876
|
|
897
|
-
## JSON
|
877
|
+
## Date and Timestamps in JSON
|
898
878
|
|
899
|
-
|
900
|
-
|
879
|
+
Elasticsearch requires dates and timestamps to have one of the formats listed
|
880
|
+
here: [https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#strict-date-time](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#strict-date-time).
|
881
|
+
|
882
|
+
However, `JSON.generate` in ruby by default outputs something like:
|
901
883
|
|
902
884
|
```ruby
|
903
|
-
|
885
|
+
JSON.generate(time: Time.now.utc)
|
886
|
+
# => "{\"time\":\"2018-02-22 18:19:33 UTC\"}"
|
904
887
|
```
|
905
888
|
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
|
889
|
+
This format is not compatible with Elasticsearch by default. If you're on
|
890
|
+
Rails, ActiveSupport adds its own `#to_json` methods to `Time`, `Date`, etc.
|
891
|
+
However, ActiveSupport checks whether they are used in combination with
|
892
|
+
`JSON.generate` or not and adapt:
|
910
893
|
|
911
894
|
```ruby
|
912
|
-
|
913
|
-
|
914
|
-
|
915
|
-
|
916
|
-
|
917
|
-
}
|
895
|
+
Time.now.utc.to_json
|
896
|
+
=> "\"2018-02-22T18:18:22.088Z\""
|
897
|
+
|
898
|
+
JSON.generate(time: Time.now.utc)
|
899
|
+
=> "{\"time\":\"2018-02-22 18:18:59 UTC\"}"
|
918
900
|
```
|
919
901
|
|
920
|
-
|
902
|
+
SearchFlip is using the [Oj gem](https://github.com/ohler55/oj) to generate
|
903
|
+
JSON. More concretely, SearchFlip is using:
|
921
904
|
|
922
905
|
```ruby
|
923
|
-
|
924
|
-
|
906
|
+
Oj.dump({ key: "value" }, mode: :custom, use_to_json: true)
|
907
|
+
```
|
925
908
|
|
926
|
-
|
927
|
-
{
|
928
|
-
# ...
|
909
|
+
This mitigates the issues if you're on Rails:
|
929
910
|
|
930
|
-
|
931
|
-
|
932
|
-
|
933
|
-
end
|
911
|
+
```ruby
|
912
|
+
Oj.dump(Time.now, mode: :custom, use_to_json: true)
|
913
|
+
# => "\"2018-02-22T18:21:21.064Z\""
|
934
914
|
```
|
935
915
|
|
936
|
-
|
916
|
+
However, if you're not on Rails, you need to add `#to_json` methods to `Time`,
|
917
|
+
`Date` and `DateTime` to get proper serialization. You can either add them on
|
918
|
+
your own, via other libraries or by simply using:
|
919
|
+
|
920
|
+
```ruby
|
921
|
+
require "search_flip/to_json"
|
922
|
+
```
|
937
923
|
|
938
924
|
## Feature Support
|
939
925
|
|
940
926
|
* for Elasticsearch 2.x, the delete-by-query plugin is required to delete
|
941
927
|
records via queries
|
942
|
-
* `#match_none` is only available with Elasticsearch >= 5
|
943
928
|
* `#track_total_hits` is only available with Elasticsearch >= 7
|
944
929
|
|
945
930
|
## Keeping your Models and Indices in Sync
|
@@ -971,6 +956,7 @@ SearchFlip is using Semantic Versioning: [SemVer](http://semver.org/)
|
|
971
956
|
|
972
957
|
* Elasticsearch: [https://www.elastic.co/](https://www.elastic.co/)
|
973
958
|
* Reference Docs: [http://www.rubydoc.info/github/mrkamel/search_flip](http://www.rubydoc.info/github/mrkamel/search_flip)
|
959
|
+
* Travis: [http://travis-ci.org/mrkamel/search_flip](http://travis-ci.org/mrkamel/search_flip)
|
974
960
|
* will_paginate: [https://github.com/mislav/will_paginate](https://github.com/mislav/will_paginate)
|
975
961
|
* kaminari: [https://github.com/kaminari/kaminari](https://github.com/kaminari/kaminari)
|
976
962
|
* Oj: [https://github.com/ohler55/oj](https://github.com/ohler55/oj)
|
data/UPDATING.md
CHANGED
@@ -1,6 +1,46 @@
|
|
1
1
|
|
2
2
|
# Updating from previous SearchFlip versions
|
3
3
|
|
4
|
+
## Update 3.x to 4.x
|
5
|
+
|
6
|
+
**[BREAKING]** For performance reasons, `SearchFlip::Result` no longer
|
7
|
+
inherits `Hashie::Mash`
|
8
|
+
|
9
|
+
* It no longer supports symbol based access like `result[:id]`
|
10
|
+
|
11
|
+
2.x:
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
CommentIndex.match_all.results.first[:id]
|
15
|
+
CommentIndex.aggregate(:tags).aggregations(:tags).values.first[:doc_count]
|
16
|
+
```
|
17
|
+
|
18
|
+
3.x
|
19
|
+
|
20
|
+
```ruby
|
21
|
+
CommentIndex.match_all.results.first["id"] # or .id
|
22
|
+
CommentIndex.aggregate(:tags).aggregations(:tags).values.first["doc_count"] # or .doc_count
|
23
|
+
```
|
24
|
+
|
25
|
+
* It no longer supports question mark methods like `result.title?`
|
26
|
+
|
27
|
+
2.x:
|
28
|
+
|
29
|
+
```ruby
|
30
|
+
CommentIndex.match_all.results.first.is_published?
|
31
|
+
```
|
32
|
+
|
33
|
+
3.x
|
34
|
+
|
35
|
+
```ruby
|
36
|
+
CommentIndex.match_all.results.first.is_published == true
|
37
|
+
```
|
38
|
+
|
39
|
+
* It no longer supports method based assignment like `result.some_key = "value"`.
|
40
|
+
|
41
|
+
However, this should not have any practical implications, as changing the
|
42
|
+
results is not considered to be a common use case.
|
43
|
+
|
4
44
|
## Update 2.x to 3.x
|
5
45
|
|
6
46
|
* **[BREAKING]** No longer pass multiple arguments to `#must`, `#must_not`,
|
data/docker-compose.yml
CHANGED
@@ -30,7 +30,7 @@ module SearchFlip
|
|
30
30
|
res[:aggregations] = aggregation_values if aggregation_values
|
31
31
|
|
32
32
|
if must_values || must_not_values || filter_values
|
33
|
-
if target.connection.
|
33
|
+
if target.connection.version.to_i >= 2
|
34
34
|
res[:filter] = {
|
35
35
|
bool: {}
|
36
36
|
.merge(must_values ? { must: must_values } : {})
|
@@ -37,6 +37,7 @@ module SearchFlip
|
|
37
37
|
}
|
38
38
|
|
39
39
|
signature_request[:body] = options[:body] if options.key?(:body)
|
40
|
+
signature_request[:body] = options[:json].respond_to?(:to_str) ? options[:json] : JSON.generate(options[:json]) if options[:json]
|
40
41
|
|
41
42
|
signature = signer.sign_request(signature_request)
|
42
43
|
|
data/lib/search_flip/bulk.rb
CHANGED
@@ -143,12 +143,12 @@ module SearchFlip
|
|
143
143
|
|
144
144
|
return if options[:raise] == false
|
145
145
|
|
146
|
-
parsed_response =
|
146
|
+
parsed_response = response.parse
|
147
147
|
|
148
148
|
return unless parsed_response["errors"]
|
149
149
|
|
150
150
|
parsed_response["items"].each do |item|
|
151
|
-
item.
|
151
|
+
item.each do |_, element|
|
152
152
|
status = element["status"]
|
153
153
|
|
154
154
|
next if status.between?(200, 299)
|
data/lib/search_flip/config.rb
CHANGED
@@ -5,12 +5,6 @@ module SearchFlip
|
|
5
5
|
bulk_limit: 1_000,
|
6
6
|
bulk_max_mb: 100,
|
7
7
|
auto_refresh: false,
|
8
|
-
instrumenter: NullInstrumenter.new
|
9
|
-
json_options: {
|
10
|
-
mode: :custom,
|
11
|
-
use_to_json: true,
|
12
|
-
time_format: :xmlschema,
|
13
|
-
bigdecimal_as_decimal: false
|
14
|
-
}
|
8
|
+
instrumenter: NullInstrumenter.new
|
15
9
|
}
|
16
10
|
end
|