elasticsearch_scanner 0.1.2 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2bb608be40ca3681cdab9f457f157eef0c0a1e84271ced6a56246425f651a4b2
-  data.tar.gz: dd0390e0fbe9ca0a44f12c45140590be3204bc11bc31aceffc03529248fa0a3f
+  metadata.gz: c3c22567dd2652dffd476fd40ed23cc43ec73e07edccf0290c650937333a0f76
+  data.tar.gz: e596d443918838c965d52cbc51580f86a850aa398c00cb38685324acecad1815
 SHA512:
-  metadata.gz: 27259fcf1dd1871a13b54b93e4b4f2fbae6755f93569d5085bd1460b48f4e42711c290882c16ef9d9399e3cbf70ede8f61ae1b41833d104e69f9b6e96a72365d
-  data.tar.gz: de0c6f9087181f1337a28cbf58b06503ad1a3062737b91e602552e815cbc02ae591021f77ff191a02af6362fdb14548ec09a2b29a516cc98e741966d272b0d20
+  metadata.gz: a5c3f8f2e2b0754f4a6ffbda29eb500731d2ca3a44bb6935f43246bd831f60fd0570863840431df7a82624b12e9a7f042cd9f1a1cf38371bba5a270235e34699
+  data.tar.gz: 6e1c4eae77bbf3dee3994e7fa10ec036fc92e00750e8dca985db84b6a53f26cebac00d08eda26eeb6e758f29bb59be9c1e3da85c624a4d84603765a9606f8661

data/elasticsearch_scanner.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'elasticsearch_scanner'
-  s.version     = '0.1.2'
+  s.version     = '0.1.5'
   s.licenses    = ['MIT']
   s.summary     = 'ElasticSearch index scanner'
   s.description = 'Iterates over the entire index'

data/lib/elasticsearch_facet_scanner.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+class ElasticSearchFacetScanner
+  include Enumerable
+
+  SCROLL_PATH = '/_search/scroll'
+
+  attr_reader :total_request_time,
+              :total_elasticsearch_time
+
+  def initialize(url, field, size=100, max_retries = 5)
+    @url = url
+    @field = field
+    @aggregation_name = "#{field}_terms"
+    @size = size
+    @max_retries = max_retries
+    @has_more = true
+    @total_request_time = 0.0
+    @total_elasticsearch_time = 0.0
+  end
+
+  def each_batch
+    yield search
+
+    while has_more?
+      yield search
+    end
+
+    nil
+  end
+
+  def each
+    each_batch do |results|
+      results.each do |result|
+        yield result
+      end
+    end
+  end
+
+  def has_more?
+    @has_more
+  end
+
+  private
+
+  def search_payload
+    {
+      size: 0,
+      aggs: {
+        @aggregation_name => {
+          terms: {
+            field: @field,
+            size: @size,
+            order: {
+              _term: :asc
+            }
+          }
+        }
+      }
+    }
+  end
+
+  def search_payload_with_range
+    search_payload.merge(
+      query: {
+        range: {
+          @field => {
+           gt:  @field_max_value
+          }
+        }
+      }
+    )
+  end
+
+  def search
+    uri = URI(@url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    req = Net::HTTP::Post.new(uri.request_uri, 'Content-Type' => 'application/json')
+    req.body = (@field_max_value ? search_payload_with_range : search_payload).to_json
+
+    make_request(http, req)
+  end
+
+  def make_request(http, req)
+    request_started_at = Time.now
+    res = http.request(req)
+    @total_request_time += (Time.now - request_started_at)
+    data = JSON.parse(res.body)
+    @total_elasticsearch_time += data['took']
+    aggregation = data['aggregations'][@aggregation_name]
+    @has_more = aggregation['sum_other_doc_count'] > 0
+    @field_max_value = aggregation['buckets'].size > 0 ? aggregation['buckets'].last['key'] : nil
+    aggregation['buckets']
+  rescue Net::ReadTimeout => e
+    attempts ||= 0
+    attempts += 1
+    if attempts < @max_retries
+      sleep([0.1, 0.2, 0.4, 1.0][attempts-1] || 1.0)
+      retry
+    end
+    raise e
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch_scanner
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.5
 platform: ruby
 authors:
 - Doug Youch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-19 00:00:00.000000000 Z
+date: 2019-10-02 00:00:00.000000000 Z
 dependencies: []
 description: Iterates over the entire index
 email: dougyouch@gmail.com
@@ -20,6 +20,7 @@ files:
 - LICENSE.txt
 - README.md
 - elasticsearch_scanner.gemspec
+- lib/elasticsearch_facet_scanner.rb
 - lib/elasticsearch_scanner.rb
 homepage: https://github.com/dougyouch/elasticsearch_scanner
 licenses: