search_flip 4.0.0.beta3 → 4.0.0.beta8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c93ffb5c3868ef14b27903734bfb2d77384acc4a52eb529070f8a17a82f323a3
-  data.tar.gz: 458f01664f5faf01006eddba2f59cdbcca7e10be22769aafca1084663c242a9b
+  metadata.gz: d63abc415f8eaad5ade7de6fc4ca424a8695b72016266a49a1d221f49530ca49
+  data.tar.gz: a6170648ed2663b769b123ae443c5d0863a4456ec39be35c2762186b93383cff
 SHA512:
-  metadata.gz: 341783040e3c705d6ac1cb7875dbbb0088efd22439057ced5f9a21c5629eb6f9b4ea7e8e4a23a0368e5ef7c249ad0e6a578a5f0f8a57afb2cb3b649b2db0a22e
-  data.tar.gz: fa40baa33a6626af128fdb14713300f55690ee407bec989b1f2bfca3e55a169377e2a0c5f9cefd6107a97ed7aceba290b52a382dcf5bdbf8c6c9389253896df7
+  metadata.gz: 7c7879bcf95f3e53c5186ec50592cf7e46b703b819c34e7d7e91229972652daf132b7a73b3f85051c6d86117f66278f83b5ac2f2b12c23272bb36130b0e7aa72
+  data.tar.gz: a04b20cbcf16eceb905798d1e5e2ea8dd60bb27b45bee9a0ab6f52aaed7d58e648a2f3139981117332a9824570db429f069ed749873dffd27ceb7cc274b3a7b4

data/.github/workflows/test.yml

@@ -11,11 +11,11 @@ jobs:
         - elasticsearch:5.4
         - docker.elastic.co/elasticsearch/elasticsearch:6.7.0
         - docker.elastic.co/elasticsearch/elasticsearch:7.0.0
-        - docker.elastic.co/elasticsearch/elasticsearch:7.9.0
+        - docker.elastic.co/elasticsearch/elasticsearch:7.11.2
         ruby:
-        - 2.5
         - 2.6
         - 2.7
+        - 3.0
     services:
       elasticsearch:
         image: ${{ matrix.elasticsearch }}
@@ -28,6 +28,7 @@ jobs:
     - uses: actions/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
+    - run: gem install bundler
     - run: bundle
     - run: sleep 10
     - run: bundle exec rspec

data/.rubocop.yml

@@ -1,6 +1,7 @@
 AllCops:
   NewCops: enable
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5
+  SuggestExtensions: false
 
 Style/CaseLikeIf:
   Enabled: false

data/CHANGELOG.md

@@ -11,6 +11,20 @@
 * Added `SearchFlip::Connection#get_cluster_settings` and
   `#update_cluster_settings`
 
+## v3.4.0
+
+* Expose `Http#timeout` via `SearchFlip::HTTPClient`
+
+## v3.3.0
+
+* Update httprb
+* Changed oj default options
+* Allow to set oj json options
+
+## v3.2.1
+
+* Fix `refresh` having a empty body breaking in elasticsearch 7.11
+
 ## v3.2.0
 
 * Fix `index_scope` not being passed in `each_record` without block

data/README.md

@@ -51,7 +51,7 @@ CommentIndex.search("hello world").where(available: true).sort(id: "desc").aggre
 
 ```
 
-Finally, SearchFlip comes with a minimal set of dependencies (http-rb and oj only).
+Finally, SearchFlip comes with a minimal set of dependencies.
 
 ## Reference Docs
 
@@ -756,7 +756,7 @@ end
 This allows to use different clusters per index e.g. when migrating indices to
 new versions of Elasticsearch.
 
-You can specify basic auth, additional headers, etc via:
+You can specify basic auth, additional headers, request timeouts, etc via:
 
 ```ruby
 http_client = SearchFlip::HTTPClient.new
@@ -773,6 +773,9 @@ http_client = http_client.via("proxy.host", 8080)
 # Custom headers
 http_client = http_client.headers(key: "value")
 
+# Timeouts
+http_client = http_client.timeout(20)
+
 SearchFlip::Connection.new(base_url: "...", http_client: http_client)
 ```
 
@@ -882,52 +885,46 @@ Thus, if your ORM supports `.find_each`, `#id` and `#where` you are already
 good to go. Otherwise, simply add your custom implementation of those methods
 that work with whatever ORM you use.
 
-## Date and Timestamps in JSON
-
-Elasticsearch requires dates and timestamps to have one of the formats listed
-here: [https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#strict-date-time](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#strict-date-time).
+## JSON
 
-However, `JSON.generate` in ruby by default outputs something like:
+SearchFlip is using the [Oj gem](https://github.com/ohler55/oj) to generate
+JSON. More concretely, SearchFlip is using:
 
 ```ruby
-JSON.generate(time: Time.now.utc)
-# => "{\"time\":\"2018-02-22 18:19:33 UTC\"}"
+Oj.dump({ key: "value" }, mode: :custom, use_to_json: true, time_format: :xmlschema, bigdecimal_as_decimal: false)
 ```
 
-This format is not compatible with Elasticsearch by default. If you're on
-Rails, ActiveSupport adds its own `#to_json` methods to `Time`, `Date`, etc.
-However, ActiveSupport checks whether they are used in combination with
-`JSON.generate` or not and adapt:
+The `use_to_json` option is used for maximum compatibility, most importantly
+when using rails `ActiveSupport::TimeWithZone` timestamps, which `oj` can not
+serialize natively. However, `use_to_json` adds performance overhead. You can
+change the json options via:
 
 ```ruby
-Time.now.utc.to_json
-=> "\"2018-02-22T18:18:22.088Z\""
-
-JSON.generate(time: Time.now.utc)
-=> "{\"time\":\"2018-02-22 18:18:59 UTC\"}"
+SearchFlip::Config[:json_options] = {
+  mode: :custom,
+  use_to_json: false,
+  time_format: :xmlschema,
+  bigdecimal_as_decimal: false
+}
 ```
 
-SearchFlip is using the [Oj gem](https://github.com/ohler55/oj) to generate
-JSON. More concretely, SearchFlip is using:
+However, you then have to convert timestamps manually for indexation via e.g.:
 
 ```ruby
-Oj.dump({ key: "value" }, mode: :custom, use_to_json: true)
-```
+class MyIndex
+  # ...
 
-This mitigates the issues if you're on Rails:
+  def self.serialize(model)
+    {
+      # ...
 
-```ruby
-Oj.dump(Time.now, mode: :custom, use_to_json: true)
-# => "\"2018-02-22T18:21:21.064Z\""
+      created_at: model.created_at.to_time
+    }
+  end
+end
 ```
 
-However, if you're not on Rails, you need to add `#to_json` methods to `Time`,
-`Date` and `DateTime` to get proper serialization. You can either add them on
-your own, via other libraries or by simply using:
-
-```ruby
-require "search_flip/to_json"
-```
+Please check out the oj docs for more details.
 
 ## Feature Support
 

data/lib/search_flip.rb

@@ -2,6 +2,7 @@ require "ruby2_keywords"
 require "forwardable"
 require "http"
 require "thread"
+require "json"
 require "oj"
 require "set"
 

data/lib/search_flip/bulk.rb

@@ -143,7 +143,7 @@ module SearchFlip
 
       return if options[:raise] == false
 
-      parsed_response = response.parse
+      parsed_response = SearchFlip::JSON.parse(response.to_s)
 
       return unless parsed_response["errors"]
 

data/lib/search_flip/config.rb

@@ -5,6 +5,12 @@ module SearchFlip
     bulk_limit: 1_000,
     bulk_max_mb: 100,
     auto_refresh: false,
-    instrumenter: NullInstrumenter.new
+    instrumenter: NullInstrumenter.new,
+    json_options: {
+      mode: :custom,
+      use_to_json: true,
+      time_format: :xmlschema,
+      bigdecimal_as_decimal: false
+    }
   }
 end

data/lib/search_flip/connection.rb

@@ -28,7 +28,9 @@ module SearchFlip
     # @return [Hash] The cluster settings
 
     def get_cluster_settings
-      http_client.get("#{base_url}/_cluster/settings").parse
+      response = http_client.get("#{base_url}/_cluster/settings")
+
+      SearchFlip::JSON.parse(response.to_s)
     end
 
     # Updates the cluster settings according to the specified payload
@@ -55,7 +57,11 @@ module SearchFlip
 
     def version
       @version_mutex.synchronize do
-        @version ||= http_client.headers(accept: "application/json").get("#{base_url}/").parse["version"]["number"]
+        @version ||= begin
+          response = http_client.headers(accept: "application/json").get("#{base_url}/")
+
+          SearchFlip::JSON.parse(response.to_s)["version"]["number"]
+        end
       end
     end
 
@@ -67,7 +73,9 @@ module SearchFlip
     # @return [Hash] The raw response
 
     def cluster_health
-      http_client.headers(accept: "application/json").get("#{base_url}/_cluster/health").parse
+      response = http_client.headers(accept: "application/json").get("#{base_url}/_cluster/health")
+
+      SearchFlip::JSON.parse(response.to_s)
     end
 
     # Uses the Elasticsearch Multi Search API to execute multiple search requests
@@ -117,10 +125,11 @@ module SearchFlip
     # @return [Hash] The raw response
 
     def update_aliases(payload)
-      http_client
+      response = http_client
         .headers(accept: "application/json", content_type: "application/json")
         .post("#{base_url}/_aliases", body: SearchFlip::JSON.generate(payload))
-        .parse
+
+      SearchFlip::JSON.parse(response.to_s)
     end
 
     # Sends an analyze request to Elasticsearch. Raises
@@ -132,10 +141,11 @@ module SearchFlip
     # @return [Hash] The raw response
 
     def analyze(request, params = {})
-      http_client
+      response = http_client
         .headers(accept: "application/json")
         .post("#{base_url}/_analyze", json: request, params: params)
-        .parse
+
+      SearchFlip::JSON.parse(response.to_s)
     end
 
     # Fetches information about the specified index aliases. Raises
@@ -151,10 +161,11 @@ module SearchFlip
     # @return [Hash] The raw response
 
     def get_aliases(index_name: "*", alias_name: "*")
-      http_client
+      response = http_client
         .headers(accept: "application/json", content_type: "application/json")
         .get("#{base_url}/#{index_name}/_alias/#{alias_name}")
-        .parse
+
+      SearchFlip::JSON.parse(response.to_s)
     end
 
     # Returns whether or not the associated Elasticsearch alias already
@@ -186,10 +197,11 @@ module SearchFlip
     # @return [Array] The raw response
 
     def get_indices(name = "*", params: {})
-      http_client
+      response = http_client
         .headers(accept: "application/json", content_type: "application/json")
         .get("#{base_url}/_cat/indices/#{name}", params: params)
-        .parse
+
+      SearchFlip::JSON.parse(response.to_s)
     end
 
     alias_method :cat_indices, :get_indices
@@ -286,10 +298,11 @@ module SearchFlip
     # @return [Hash] The index settings
 
     def get_index_settings(index_name)
-      http_client
+      response = http_client
         .headers(accept: "application/json")
         .get("#{index_url(index_name)}/_settings")
-        .parse
+
+      SearchFlip::JSON.parse(response.to_s)
     end
 
     # Sends a refresh request to Elasticsearch. Raises
@@ -299,7 +312,7 @@ module SearchFlip
     # @return [Boolean] Returns true or raises SearchFlip::ResponseError
 
     def refresh(index_names = nil)
-      http_client.post("#{index_names ? index_url(Array(index_names).join(",")) : base_url}/_refresh", json: {})
+      http_client.post("#{index_names ? index_url(Array(index_names).join(",")) : base_url}/_refresh")
 
       true
     end
@@ -337,7 +350,9 @@ module SearchFlip
       url = type_name ? type_url(index_name, type_name) : index_url(index_name)
       params = type_name && version.to_f >= 6.7 ? { include_type_name: true } : {}
 
-      http_client.headers(accept: "application/json").get("#{url}/_mapping", params: params).parse
+      response = http_client.headers(accept: "application/json").get("#{url}/_mapping", params: params)
+
+      SearchFlip::JSON.parse(response.to_s)
     end
 
     # Deletes the specified index from Elasticsearch. Raises

data/lib/search_flip/http_client.rb

@@ -1,7 +1,28 @@
 module SearchFlip
-  # The SearchFlip::HTTPClient class wraps the http gem, is for internal use
-  # and responsible for the http request/response handling, ie communicating
-  # with Elasticsearch.
+  # The SearchFlip::HTTPClient class wraps the http gem and responsible for the
+  # http request/response handling, ie communicating with Elasticsearch. You
+  # only need to use it directly if you need authentication to communicate with
+  # Elasticsearch or if you want to set some custom http settings.
+  #
+  # @example
+  #   http_client = SearchFlip::HTTPClient.new
+  #
+  #   # Basic Auth
+  #   http_client = http_client.basic_auth(user: "username", pass: "password")
+  #
+  #   # Raw Auth Header
+  #   http_client = http_client.auth("Bearer VGhlIEhUVFAgR2VtLCBST0NLUw")
+  #
+  #   # Proxy Settings
+  #   http_client = http_client.via("proxy.host", 8080)
+  #
+  #   # Custom headers
+  #   http_client = http_client.headers(key: "value")
+  #
+  #   # Timeouts
+  #   http_client = http_client.timeout(20)
+  #
+  #   SearchFlip::Connection.new(base_url: "...", http_client: http_client)
 
   class HTTPClient
     attr_accessor :request, :plugins
@@ -14,11 +35,11 @@ module SearchFlip
     class << self
       extend Forwardable
 
-      def_delegators :new, :headers, :via, :basic_auth, :auth
+      def_delegators :new, :headers, :via, :basic_auth, :auth, :timeout
       def_delegators :new, :get, :post, :put, :delete, :head
     end
 
-    [:headers, :via, :basic_auth, :auth].each do |method|
+    [:headers, :via, :basic_auth, :auth, :timeout].each do |method|
       define_method method do |*args|
         dup.tap do |client|
           client.request = request.send(method, *args)

data/lib/search_flip/index.rb

@@ -487,7 +487,9 @@ module SearchFlip
       # @return [Hash] The raw response
 
       def analyze(request, params = {})
-        connection.http_client.headers(accept: "application/json").post("#{index_url}/_analyze", json: request, params: params).parse
+        response = connection.http_client.headers(accept: "application/json").post("#{index_url}/_analyze", json: request, params: params)
+
+        SearchFlip::JSON.parse(response.to_s)
       end
 
       # Sends a index refresh request to Elasticsearch. Raises

data/lib/search_flip/json.rb

@@ -1,11 +1,11 @@
 module SearchFlip
   class JSON
     def self.generate(obj)
-      Oj.dump(obj, mode: :custom, use_to_json: true)
+      Oj.dump(obj, SearchFlip::Config[:json_options])
     end
 
-    def self.parse(str)
-      Oj.load(str, mode: :compat)
+    def self.parse(json)
+      ::JSON.parse(json)
     end
   end
 end

data/lib/search_flip/to_json.rb

@@ -1,29 +1 @@
-require "time"
-require "date"
-require "json"
-
-class Time
-  def to_json(*args)
-    iso8601(6).to_json
-  end
-end
-
-class Date
-  def to_json(*args)
-    iso8601.to_json
-  end
-end
-
-class DateTime
-  def to_json(*args)
-    iso8601(6).to_json
-  end
-end
-
-if defined?(ActiveSupport)
-  class ActiveSupport::TimeWithZone
-    def to_json(*args)
-      iso8601(6).to_json
-    end
-  end
-end
+warn "[DEPRECATION] Using search_flip/to_json is not neccessary anymore"

data/lib/search_flip/version.rb

@@ -1,3 +1,3 @@
 module SearchFlip
-  VERSION = "4.0.0.beta3"
+  VERSION = "4.0.0.beta8"
 end

data/search_flip.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock"
 
   spec.add_dependency "http"
+  spec.add_dependency "json"
   spec.add_dependency "oj"
   spec.add_dependency "ruby2_keywords"
 end

data/spec/search_flip/http_client_spec.rb

@@ -7,7 +7,7 @@ class HttpTestRequest
     self.calls = []
   end
 
-  [:via, :basic_auth, :auth].each do |method|
+  [:headers, :via, :basic_auth, :auth, :timeout].each do |method|
     define_method method do |*args|
       dup.tap do |request|
         request.calls = calls + [[method, args]]
@@ -20,7 +20,7 @@ RSpec.describe SearchFlip::HTTPClient do
   describe "delegation" do
     subject { SearchFlip::HTTPClient }
 
-    [:headers, :via, :basic_auth, :auth].each do |method|
+    [:headers, :via, :basic_auth, :auth, :timeout].each do |method|
       it { should delegate(method).to(:new) }
     end
 
@@ -56,8 +56,12 @@ RSpec.describe SearchFlip::HTTPClient do
     end
   end
 
-  [:via, :basic_auth, :auth].each do |method|
+  [:headers, :via, :basic_auth, :auth, :timeout].each do |method|
     describe "##{method}" do
+      it "is understood by HTTP" do
+        expect(HTTP.respond_to?(method)).to eq(true)
+      end
+
       it "creates a dupped instance" do
         client = SearchFlip::HTTPClient.new
         client.request = HttpTestRequest.new

data/spec/search_flip/json_spec.rb

@@ -2,6 +2,16 @@ require File.expand_path("../spec_helper", __dir__)
 
 RSpec.describe SearchFlip::JSON do
   describe ".generate" do
+    it "encodes timestamps correctly" do
+      Timecop.freeze "2020-06-01 12:00:00 UTC" do
+        expect(described_class.generate(timestamp: Time.now.utc)).to eq('{"timestamp":"2020-06-01T12:00:00.000Z"}')
+      end
+    end
+
+    it "encodes bigdecimals as string" do
+      expect(described_class.generate(value: BigDecimal(1))).to eq('{"value":"1.0"}')
+    end
+
     it "delegates to Oj" do
       allow(Oj).to receive(:dump)
 
@@ -9,7 +19,7 @@ RSpec.describe SearchFlip::JSON do
 
       described_class.generate(payload)
 
-      expect(Oj).to have_received(:dump).with(payload, mode: :custom, use_to_json: true)
+      expect(Oj).to have_received(:dump).with(payload, mode: :custom, use_to_json: true, time_format: :xmlschema, bigdecimal_as_decimal: false)
     end
 
     it "generates json" do
@@ -18,14 +28,18 @@ RSpec.describe SearchFlip::JSON do
   end
 
   describe ".parse" do
-    it "delegates to Oj" do
-      allow(Oj).to receive(:load)
+    it "returns the parsed json payload" do
+      expect(described_class.parse('{"key":"value"}')).to eq("key" => "value")
+    end
+
+    it "delegates to JSON" do
+      allow(JSON).to receive(:parse)
 
       payload = '{"key":"value"}'
 
       described_class.parse(payload)
 
-      expect(Oj).to have_received(:load).with(payload, mode: :compat)
+      expect(JSON).to have_received(:parse).with(payload)
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: search_flip
 version: !ruby/object:Gem::Version
-  version: 4.0.0.beta3
+  version: 4.0.0.beta8
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-05 00:00:00.000000000 Z
+date: 2021-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
@@ -250,7 +264,6 @@ files:
 - spec/search_flip/null_instrumenter_spec.rb
 - spec/search_flip/response_spec.rb
 - spec/search_flip/result_spec.rb
-- spec/search_flip/to_json_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/mrkamel/search_flip
 licenses:
@@ -292,5 +305,4 @@ test_files:
 - spec/search_flip/null_instrumenter_spec.rb
 - spec/search_flip/response_spec.rb
 - spec/search_flip/result_spec.rb
-- spec/search_flip/to_json_spec.rb
 - spec/spec_helper.rb

data/spec/search_flip/to_json_spec.rb

@@ -1,28 +0,0 @@
-require File.expand_path("../spec_helper", __dir__)
-require "search_flip/to_json"
-
-RSpec.describe "to_json" do
-  it "uses the correct format for Time" do
-    Timecop.freeze Time.parse("2018-01-01 12:00:00 UTC") do
-      expect(Time.now.utc.to_json).to eq("\"2018-01-01T12:00:00.000000Z\"")
-    end
-  end
-
-  it "uses the correct format for Date" do
-    Timecop.freeze Time.parse("2018-01-01 12:00:00 UTC") do
-      expect(Date.today.to_json).to eq("\"2018-01-01\"")
-    end
-  end
-
-  it "uses the correct format for DateTime" do
-    Timecop.freeze Time.parse("2018-01-01 12:00:00 UTC") do
-      expect(Time.now.utc.to_json).to eq("\"2018-01-01T12:00:00.000000Z\"")
-    end
-  end
-
-  it "uses the correct format for TimeWithZone" do
-    Timecop.freeze Time.parse("2018-01-01 12:00:00 UTC") do
-      expect(Time.find_zone("UTC").now.to_json).to eq("\"2018-01-01T12:00:00.000000Z\"")
-    end
-  end
-end