search_flip 2.0.0.beta4 → 2.0.0.beta5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c626b4c449c70b071fee204d164b226271d2f098ae48f4c54568e07b6ad8d9d8
-  data.tar.gz: f35599e1cd57a27fe3c008fa20538be08fa9f7251a5c421d25fd98113d2a6cc7
+  metadata.gz: b0d1ed2d2431d99cbe9949cb24154bfda3c2188713eb7432a6a8f1c6d5e45fe8
+  data.tar.gz: fbb8f6f7989552d6944a3d238ab3ca22c5ff2ca2c2c3ca7dddf1a3d8c4825ecc
 SHA512:
-  metadata.gz: 95df5e882633308f0dcbcac35d155901db7bfee39f0f37d1ee84c055fdde821a22b697fd0df5ea4ab82204896c3adb483769cae85b1489ec2eca28c4fc0e2de5
-  data.tar.gz: e97fceeacf0186b99829fb70d9a8eb396455c2f2cf3dec58e2f357dd5b8bd542025e7ee33201757af02c45b90752da70e6d28e6cbff2605fc36d0fd7ac9638d2
+  metadata.gz: 2f72caff2b7806d19d88d31ea8900fa739d10975d121d83245e7111552cbfc95fda81dde504f8b235ed4fdc6a9a6a3056029c404a638045d86a6c9fc606a23fd
+  data.tar.gz: ea91ff174b46ad96bb29a94a9173ba9295ec503e731abc6e0956b1788dd73197aebc75c43eb302f59c6554822c5ebf2c7dee6006878462137ad1e57b1c512dad

data/README.md

@@ -705,7 +705,7 @@ you can e.g. use docker-compose:
 ```
 $ cd search_flip
 $ sudo ES_IMAGE=elasticsearch:5.4 docker-compose up
-$ rake test
+$ rspec
 ```
 
 That's it.

data/lib/search_flip/bulk.rb

@@ -14,7 +14,7 @@ module SearchFlip
   class Bulk
     class Error < StandardError; end
 
-    attr_accessor :url, :count, :options, :http_client, :ignore_errors
+    attr_reader :url, :options, :ignore_errors
 
     # Builds and yields a new Bulk object, ie initiates the buffer, yields,
     # sends batches of records each time the buffer is full, and sends a final
@@ -32,8 +32,9 @@ module SearchFlip
     #   end
     #
     # @param url [String] The endpoint to send bulk requests to
-    # @param count [Fixnum] The maximum number of documents per bulk request
     # @param options [Hash] Options for the bulk requests
+    # @option options batch_size [Fixnum] The maximum number of documents per bulk
+    #   request
     # @option options ignore_errors [Array, Fixnum] Errors that should be
     #   ignored. If you eg want to ignore errors resulting from conflicts,
     #   you can specify to ignore 409 here.
@@ -43,12 +44,16 @@ module SearchFlip
     # @option options http_client [SearchFlip::HTTPClient] An optional http
     #   client instance
 
-    def initialize(url, count = 1_000, options = {})
-      self.url = url
-      self.count = count
-      self.options = options
-      self.http_client = options[:http_client] || SearchFlip::HTTPClient.new
-      self.ignore_errors = Array(options[:ignore_errors]).to_set if options[:ignore_errors]
+    def initialize(url, options = {})
+      @url = url
+      @options = options
+      @http_client = options[:http_client] || SearchFlip::HTTPClient.new
+      @ignore_errors = Array(options[:ignore_errors]).to_set if options[:ignore_errors]
+
+      @bulk_limit = options[:bulk_limit] || SearchFlip::Config[:bulk_limit]
+      @bulk_max_mb = options[:bulk_max_mb] || SearchFlip::Config[:bulk_max_mb]
+
+      @bulk_max_bytes = @bulk_max_mb * 1024 * 1024
 
       init
 
@@ -117,7 +122,7 @@ module SearchFlip
 
     def upload
       response =
-        http_client
+        @http_client
           .headers(accept: "application/json", content_type: "application/x-ndjson")
           .put(url, body: @payload, params: ignore_errors ? {} : { filter_path: "errors" })
 
@@ -141,17 +146,21 @@ module SearchFlip
     end
 
     def perform(action, id, json = nil, options = {})
-      @payload << SearchFlip::JSON.generate(action => options.merge(_id: id))
-      @payload << "\n"
+      new_payload = SearchFlip::JSON.generate(action => options.merge(_id: id))
+      new_payload << "\n"
 
       if json
-        @payload << json
-        @payload << "\n"
+        new_payload << json
+        new_payload << "\n"
       end
 
+      upload if @num > 0 && @payload.bytesize + new_payload.bytesize >= @bulk_max_bytes
+
+      @payload << new_payload
+
       @num += 1
 
-      upload if @num >= count
+      upload if @num >= @bulk_limit || @payload.bytesize >= @bulk_max_bytes
     end
   end
 end

data/lib/search_flip/config.rb

@@ -4,6 +4,7 @@ module SearchFlip
     index_prefix: nil,
     base_url: "http://127.0.0.1:9200",
     bulk_limit: 1_000,
+    bulk_max_mb: 100,
     auto_refresh: false
   }
 end

data/lib/search_flip/connection.rb

@@ -1,19 +1,22 @@
 
 module SearchFlip
   class Connection
-    attr_reader :base_url, :http_client
+    attr_reader :base_url, :http_client, :bulk_limit, :bulk_max_mb
 
     # Creates a new connection.
     #
     # @example
     #   SearchFlip::Connection.new(base_url: "http://elasticsearch.host:9200")
     #
-    # @param base_url [String] The base url for the connection
-    # @param http_client [SearchFlip::HTTPClient] An optional http client instance
+    # @param options [Hash] A hash containing the config options
+    # @option options base_url [String] The base url for the connection
+    # @option options http_client [SearchFlip::HTTPClient] An optional http client instance
+    # @option options bulk_max_mb [Fixnum] An optional MB limit for bulk requests
 
-    def initialize(base_url: SearchFlip::Config[:base_url], http_client: SearchFlip::HTTPClient.new)
-      @base_url = base_url
-      @http_client = http_client
+    def initialize(options = {})
+      @base_url = options[:base_url] || SearchFlip::Config[:base_url]
+      @http_client = options[:http_client] || SearchFlip::HTTPClient.new
+      @bulk_limit = options[:bulk_limit] || SearchFlip::Config[:bulk_limit]
     end
 
     # Queries and returns the ElasticSearch version used.

data/lib/search_flip/index.rb

@@ -538,9 +538,13 @@ module SearchFlip
       #   raise.
 
       def bulk(options = {})
-        opts = { http_client: connection.http_client }.merge(options)
+        default_options = {
+          http_client: connection.http_client,
+          bulk_limit: connection.bulk_limit,
+          bulk_max_mb: connection.bulk_max_mb
+        }
 
-        SearchFlip::Bulk.new("#{type_url}/_bulk", SearchFlip::Config[:bulk_limit], opts) do |indexer|
+        SearchFlip::Bulk.new("#{type_url}/_bulk", default_options.merge(options)) do |indexer|
           yield indexer
         end
 

data/lib/search_flip/version.rb

@@ -1,5 +1,5 @@
 
 module SearchFlip
-  VERSION = "2.0.0.beta4"
+  VERSION = "2.0.0.beta5"
 end
 

data/spec/search_flip/bulk_spec.rb

@@ -74,5 +74,31 @@ RSpec.describe SearchFlip::Bulk do
 
       expect(&block).to raise_error(SearchFlip::ResponseError)
     end
+
+    it "handles overly long payloads" do
+      product = create(:product)
+
+      allow(product).to receive(:description).and_return("x" * 1024 * 1024 * 10)
+
+      block = lambda do
+        ProductIndex.bulk bulk_max_mb: 1_000 do |bulk|
+          100.times do
+            bulk.index product.id, ProductIndex.serialize(product)
+          end
+        end
+      end
+
+      expect(&block).to raise_error(SearchFlip::ResponseError)
+
+      block = lambda do
+        ProductIndex.bulk bulk_max_mb: 100 do |bulk|
+          100.times do
+            bulk.index product.id, ProductIndex.serialize(product)
+          end
+        end
+      end
+
+      expect(&block).not_to raise_error #(SearchFlip::ResponseError)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: search_flip
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta4
+  version: 2.0.0.beta5
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-15 00:00:00.000000000 Z
+date: 2019-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord