elasticsearch_scanner 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0c430016b9eba6aecdffecc2da894a83735979d58c74f0a0f6f1142de91db67f
+  data.tar.gz: 4d0279810b2b5306524101d7dbd5ee70c2ce63f7ddda5b155019b60d9bd3ded3
+SHA512:
+  metadata.gz: 7eb2d0443e03e24ff1d9021ee6e1883069e6281e57a22416b427fa9efc75b7c3ecb443112da319d82b11c44d10a86b1307dc9cdb423512e66c7542ea510709f6
+  data.tar.gz: 015d278b62b07893b32383d8b6d3b286ac6e968ca203b01c920588c8832d9a5127eecda4ef5461d9de62d987c46dee55a2186467424840500bb6cf9464cca530

data/.gitignore

@@ -0,0 +1,51 @@
+# rcov generated
+coverage
+coverage.data
+
+# rdoc generated
+rdoc
+
+# yard generated
+doc
+.yardoc
+
+# bundler
+.bundle
+
+# jeweler generated
+pkg
+
+# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
+#
+# * Create a file at ~/.gitignore
+# * Include files you want ignored
+# * Run: git config --global core.excludesfile ~/.gitignore
+#
+# After doing this, these files will be ignored in all your git projects,
+# saving you from having to 'pollute' every project you touch with them
+#
+# Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
+#
+# For MacOS:
+#
+.DS_Store
+
+# For TextMate
+#*.tmproj
+#tmtags
+
+# For emacs:
+*~
+\#*
+.\#*
+
+# For vim:
+*.swp
+
+# For redcar:
+#.redcar
+
+# For rubinius:
+#*.rbc
+
+*.gem

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Douglas Youch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1 @@
+# elasticsearch_scanner

data/elasticsearch_scanner.gemspec

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |s|
+  s.name        = 'elasticsearch_scanner'
+  s.version     = '0.1.0'
+  s.licenses    = ['MIT']
+  s.summary     = 'ElasticSearch index scanner'
+  s.description = 'Iterates over the entire index'
+  s.authors     = ['Doug Youch']
+  s.email       = 'dougyouch@gmail.com'
+  s.homepage    = 'https://github.com/dougyouch/elasticsearch_scanner'
+  s.files       = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+end

data/lib/elasticsearch_scanner.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+class ElasticSearchScanner
+  include Enumerable
+
+  SCROLL_PATH = '/_search/scroll'
+
+  attr_reader :total_request_time,
+              :total_elasticsearch_time
+
+  def initialize(url, query, size=100, scroll_ttl='1m')
+    @url = url
+    @query = query
+    @size = size
+    @scroll_ttl = scroll_ttl
+    @fields_to_return = true # all fields
+    @has_more = true
+    @total_request_time = 0.0
+    @total_elasticsearch_time = 0.0
+  end
+
+  def fields_to_return=(fields_to_return)
+    @fields_to_return = fields_to_return
+  end
+
+  def each_batch
+    yield search
+
+    while has_more?
+      yield scroll
+    end
+
+    nil
+  end
+
+  def each
+    each_batch do |results|
+      results.each do |result|
+        yield result
+      end
+    end
+  end
+
+  def has_more?
+    @has_more
+  end
+
+  private
+
+  def search_payload
+    {
+      size: @size,
+      _source: @fields_to_return,
+      query: @query
+    }
+  end
+
+  def search
+    uri = URI(@url + '?scroll=' + @scroll_ttl)
+    http = Net::HTTP.new(uri.host, uri.port)
+    req = Net::HTTP::Post.new(uri.request_uri, 'Content-Type' => 'application/json')
+    req.body = search_payload.to_json
+
+    make_request(http, req)
+  end
+
+  def scroll
+    uri = URI(@url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    req = Net::HTTP::Post.new(SCROLL_PATH, 'Content-Type' => 'application/json')
+    req.body = {
+      scroll: @scroll_ttl,
+      scroll_id: @scroll_id
+    }.to_json
+
+    make_request(http, req)
+  end
+
+  def make_request(http, req)
+    request_started_at = Time.now
+    res = http.request(req)
+    @total_request_time += (Time.now - request_started_at)
+    data = JSON.parse(res.body)
+    @total_elasticsearch_time += data['took']
+    @scroll_id = data['_scroll_id']
+    @has_more = data['hits']['hits'].size == @size
+    data['hits']['hits']
+  end
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: elasticsearch_scanner
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Doug Youch
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-09-19 00:00:00.000000000 Z
+dependencies: []
+description: Iterates over the entire index
+email: dougyouch@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- LICENSE.txt
+- README.md
+- elasticsearch_scanner.gemspec
+- lib/elasticsearch_scanner.rb
+homepage: https://github.com/dougyouch/elasticsearch_scanner
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: ElasticSearch index scanner
+test_files: []