metacrunch-elasticsearch 2.1.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +9 -15
- data/lib/metacrunch/elasticsearch.rb +0 -2
- data/lib/metacrunch/elasticsearch/index_creator.rb +3 -4
- data/lib/metacrunch/elasticsearch/indexer.rb +4 -5
- data/lib/metacrunch/elasticsearch/searcher.rb +4 -10
- data/lib/metacrunch/elasticsearch/version.rb +1 -1
- data/metacrunch-elasticsearch.gemspec +2 -3
- metadata +2 -20
- data/lib/metacrunch/elasticsearch/cli.rb +0 -59
- data/lib/metacrunch/elasticsearch/cli/dump_metadata.rb +0 -84
- data/lib/metacrunch/elasticsearch/cli/list_indices.rb +0 -19
- data/lib/metacrunch_plugin.rb +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3c8e449f5c84fd5b22946dc75c52b1693830e6c4
|
4
|
+
data.tar.gz: b698c9cc5904719a37d6902e4cee9528ea03577e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 396ae218ce4e493aa8d96bc7a5061e5941a65ea56c9d8cd791a9bcaeb0de11f5f83fc1df5f97379f967d969c86626b084dd9d39ed6fbeb3e6dbed7aacf88f2ae
|
7
|
+
data.tar.gz: 4254f096d00cb3d113e43f5b5b9e215699002972f6661a335a1c4683bdcaf76bcbe12128d2a45b30e8a4bb5043138a16887f6c049d6c52db9fe3e1af23e5cebe
|
data/Gemfile
CHANGED
@@ -1,27 +1,21 @@
|
|
1
1
|
source "https://rubygems.org"
|
2
2
|
|
3
|
-
# Specify your gem's dependencies in your gemspec
|
4
3
|
gemspec
|
5
4
|
|
6
5
|
group :development do
|
7
|
-
gem "bundler",
|
8
|
-
gem "rake"
|
9
|
-
gem "rspec",
|
10
|
-
gem "simplecov",
|
11
|
-
gem "vcr", ">= 2.9.0", "< 3.0.0"
|
12
|
-
gem "webmock", ">= 1.19.0", "< 2.0.0"
|
6
|
+
gem "bundler", ">= 1.7"
|
7
|
+
gem "rake", ">= 11.1"
|
8
|
+
gem "rspec", ">= 3.0.0", "< 4.0.0"
|
9
|
+
gem "simplecov", ">= 0.11.0"
|
13
10
|
|
14
11
|
if !ENV["CI"]
|
15
|
-
gem "hashdiff"
|
16
|
-
gem "pry",
|
17
|
-
gem "pry-
|
18
|
-
gem "pry-
|
19
|
-
gem "pry-state", "~> 0.1.7"
|
12
|
+
gem "hashdiff", ">= 0.3.0", platform: :ruby
|
13
|
+
gem "pry-byebug", ">= 3.3.0", platform: :ruby
|
14
|
+
gem "pry-rescue", ">= 1.4.2", platform: :ruby
|
15
|
+
gem "pry-state", ">= 0.1.7", platform: :ruby
|
20
16
|
end
|
21
17
|
end
|
22
18
|
|
23
19
|
group :test do
|
24
|
-
gem "codeclimate-test-reporter", require: nil
|
20
|
+
gem "codeclimate-test-reporter", ">= 0.5.0", require: nil
|
25
21
|
end
|
26
|
-
|
27
|
-
gem "metacrunch", github: "ubpb/metacrunch", branch: :master
|
@@ -1,9 +1,7 @@
|
|
1
|
-
require "metacrunch"
|
2
1
|
require "elasticsearch"
|
3
2
|
|
4
3
|
module Metacrunch
|
5
4
|
module Elasticsearch
|
6
|
-
require_relative "./elasticsearch/cli"
|
7
5
|
require_relative "./elasticsearch/index_creator"
|
8
6
|
require_relative "./elasticsearch/indexer"
|
9
7
|
require_relative "./elasticsearch/reader"
|
@@ -1,10 +1,9 @@
|
|
1
1
|
require "elasticsearch"
|
2
|
-
require "metacrunch/processor"
|
3
2
|
require_relative "../elasticsearch"
|
4
3
|
require_relative "./client_factory"
|
5
4
|
require_relative "./options_helpers"
|
6
5
|
|
7
|
-
class Metacrunch::Elasticsearch::IndexCreator
|
6
|
+
class Metacrunch::Elasticsearch::IndexCreator
|
8
7
|
include Metacrunch::Elasticsearch::ClientFactory
|
9
8
|
include Metacrunch::Elasticsearch::OptionsHelpers
|
10
9
|
|
@@ -19,9 +18,9 @@ class Metacrunch::Elasticsearch::IndexCreator < Metacrunch::Processor
|
|
19
18
|
raise ArgumentError.new("You have to supply an index name!") if @client_args[:index].blank?
|
20
19
|
end
|
21
20
|
|
22
|
-
def call(items = []
|
21
|
+
def call(items = [])
|
23
22
|
client = client_factory
|
24
|
-
logger =
|
23
|
+
logger = @logger
|
25
24
|
|
26
25
|
if client.indices.exists?(@client_args)
|
27
26
|
if @delete_existing_index == true
|
@@ -1,10 +1,9 @@
|
|
1
1
|
require "elasticsearch"
|
2
|
-
require "metacrunch/processor"
|
3
2
|
require_relative "../elasticsearch"
|
4
3
|
require_relative "./client_factory"
|
5
4
|
require_relative "./options_helpers"
|
6
5
|
|
7
|
-
class Metacrunch::Elasticsearch::Indexer
|
6
|
+
class Metacrunch::Elasticsearch::Indexer
|
8
7
|
include Metacrunch::Elasticsearch::ClientFactory
|
9
8
|
include Metacrunch::Elasticsearch::OptionsHelpers
|
10
9
|
|
@@ -14,15 +13,15 @@ class Metacrunch::Elasticsearch::Indexer < Metacrunch::Processor
|
|
14
13
|
attr_accessor :index
|
15
14
|
attr_accessor :logger
|
16
15
|
attr_accessor :type
|
17
|
-
|
16
|
+
|
18
17
|
def initialize(options = {})
|
19
18
|
(@client_args = options).deep_symbolize_keys!
|
20
19
|
extract_options!(@client_args, :_client_options_, :bulk_size, :callbacks, :id_accessor, :index, :logger, :type)
|
21
20
|
raise ArgumentError.new("You have to supply an index name!") if @index.blank?
|
22
21
|
end
|
23
22
|
|
24
|
-
def call(items = []
|
25
|
-
logger =
|
23
|
+
def call(items = [])
|
24
|
+
logger = @logger
|
26
25
|
|
27
26
|
if (slice_size = @bulk_size || items.length) > 0
|
28
27
|
client = client_factory
|
@@ -1,10 +1,9 @@
|
|
1
1
|
require "elasticsearch"
|
2
|
-
require "metacrunch/processor"
|
3
2
|
require_relative "../elasticsearch"
|
4
3
|
require_relative "./client_factory"
|
5
4
|
require_relative "./options_helpers"
|
6
5
|
|
7
|
-
class Metacrunch::Elasticsearch::Searcher
|
6
|
+
class Metacrunch::Elasticsearch::Searcher
|
8
7
|
include Enumerable
|
9
8
|
include Metacrunch::Elasticsearch::ClientFactory
|
10
9
|
include Metacrunch::Elasticsearch::OptionsHelpers
|
@@ -18,21 +17,16 @@ class Metacrunch::Elasticsearch::Searcher < Metacrunch::Processor
|
|
18
17
|
attr_accessor :scan_size
|
19
18
|
attr_accessor :scroll_expiry_time
|
20
19
|
attr_accessor :type
|
21
|
-
|
20
|
+
|
22
21
|
def initialize(options = {})
|
23
22
|
options.deep_symbolize_keys!
|
24
23
|
extract_options!(options, :_client_options_, :bulk_size, :index, :scan_size, :scroll_expiry_time, :type)
|
25
24
|
@body = options.presence || DEFAULT_BODY
|
26
25
|
end
|
27
26
|
|
28
|
-
def call(items = []
|
27
|
+
def call(items = [])
|
29
28
|
@docs_enumerator ||= @bulk_size ? each_slice(@bulk_size) : [each.to_a].to_enum
|
30
|
-
|
31
|
-
begin
|
32
|
-
items.concat(@docs_enumerator.next)
|
33
|
-
rescue StopIteration
|
34
|
-
pipeline.terminate!
|
35
|
-
end
|
29
|
+
items.concat(@docs_enumerator.next)
|
36
30
|
end
|
37
31
|
|
38
32
|
def each
|
@@ -16,8 +16,7 @@ Gem::Specification.new do |spec|
|
|
16
16
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
17
17
|
spec.require_paths = ["lib"]
|
18
18
|
|
19
|
-
spec.add_dependency "activesupport",
|
20
|
-
spec.add_dependency "elasticsearch",
|
21
|
-
spec.add_dependency "metacrunch", "~> 2.1"
|
19
|
+
spec.add_dependency "activesupport", ">= 4.0.0"
|
20
|
+
spec.add_dependency "elasticsearch", "~> 1.0"
|
22
21
|
end
|
23
22
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: metacrunch-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- René Sprotte
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-
|
12
|
+
date: 2016-05-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|
@@ -39,20 +39,6 @@ dependencies:
|
|
39
39
|
- - "~>"
|
40
40
|
- !ruby/object:Gem::Version
|
41
41
|
version: '1.0'
|
42
|
-
- !ruby/object:Gem::Dependency
|
43
|
-
name: metacrunch
|
44
|
-
requirement: !ruby/object:Gem::Requirement
|
45
|
-
requirements:
|
46
|
-
- - "~>"
|
47
|
-
- !ruby/object:Gem::Version
|
48
|
-
version: '2.1'
|
49
|
-
type: :runtime
|
50
|
-
prerelease: false
|
51
|
-
version_requirements: !ruby/object:Gem::Requirement
|
52
|
-
requirements:
|
53
|
-
- - "~>"
|
54
|
-
- !ruby/object:Gem::Version
|
55
|
-
version: '2.1'
|
56
42
|
description:
|
57
43
|
email:
|
58
44
|
executables: []
|
@@ -69,9 +55,6 @@ files:
|
|
69
55
|
- bin/console
|
70
56
|
- bin/setup
|
71
57
|
- lib/metacrunch/elasticsearch.rb
|
72
|
-
- lib/metacrunch/elasticsearch/cli.rb
|
73
|
-
- lib/metacrunch/elasticsearch/cli/dump_metadata.rb
|
74
|
-
- lib/metacrunch/elasticsearch/cli/list_indices.rb
|
75
58
|
- lib/metacrunch/elasticsearch/client_factory.rb
|
76
59
|
- lib/metacrunch/elasticsearch/index_creator.rb
|
77
60
|
- lib/metacrunch/elasticsearch/indexer.rb
|
@@ -81,7 +64,6 @@ files:
|
|
81
64
|
- lib/metacrunch/elasticsearch/uri.rb
|
82
65
|
- lib/metacrunch/elasticsearch/version.rb
|
83
66
|
- lib/metacrunch/elasticsearch/writer.rb
|
84
|
-
- lib/metacrunch_plugin.rb
|
85
67
|
- metacrunch-elasticsearch.gemspec
|
86
68
|
homepage: http://github.com/ubpb/metacrunch-elasticsearch
|
87
69
|
licenses:
|
@@ -1,59 +0,0 @@
|
|
1
|
-
module Metacrunch
|
2
|
-
module Elasticsearch
|
3
|
-
module Cli
|
4
|
-
require_relative "./cli/dump_metadata"
|
5
|
-
require_relative "./cli/list_indices"
|
6
|
-
#require_relative "./cli/reindex_command"
|
7
|
-
end
|
8
|
-
end
|
9
|
-
end
|
10
|
-
|
11
|
-
Metacrunch::Cli.setup("elasticsearch", "Commands for Elasticsearch") do |r|
|
12
|
-
#
|
13
|
-
#
|
14
|
-
#
|
15
|
-
r.register(Metacrunch::Elasticsearch::Cli::DumpMetadata) do |c|
|
16
|
-
c.name "dump_metadata"
|
17
|
-
c.usage "dump_metadata"
|
18
|
-
c.desc "Dump index metadata"
|
19
|
-
|
20
|
-
c.option :url,
|
21
|
-
desc: "Elasticsearch url",
|
22
|
-
type: :string,
|
23
|
-
aliases: "-u",
|
24
|
-
required: true
|
25
|
-
c.option :format,
|
26
|
-
desc: "Output format",
|
27
|
-
type: :string,
|
28
|
-
aliases: "-f",
|
29
|
-
default: :yaml
|
30
|
-
c.option :mappings_filename,
|
31
|
-
desc: "Dump mappings",
|
32
|
-
aliases: "-m"
|
33
|
-
c.option :settings_filename,
|
34
|
-
desc: "Dump settings",
|
35
|
-
aliases: "-s",
|
36
|
-
default: false
|
37
|
-
c.option :index,
|
38
|
-
desc: "Index name",
|
39
|
-
aliases: "-i",
|
40
|
-
required: true
|
41
|
-
c.option :type,
|
42
|
-
desc: "Type to dump mapping for",
|
43
|
-
aliases: "-t",
|
44
|
-
default: "_default_"
|
45
|
-
|
46
|
-
end
|
47
|
-
|
48
|
-
r.register(Metacrunch::Elasticsearch::Cli::ListIndices) do |c|
|
49
|
-
c.name "list_indices"
|
50
|
-
c.usage "list_indices"
|
51
|
-
c.desc "List indices"
|
52
|
-
|
53
|
-
c.option :url,
|
54
|
-
desc: "Elasticsearch url",
|
55
|
-
type: :string,
|
56
|
-
aliases: "-u",
|
57
|
-
required: true
|
58
|
-
end
|
59
|
-
end
|
@@ -1,84 +0,0 @@
|
|
1
|
-
require "elasticsearch"
|
2
|
-
require_relative "../cli"
|
3
|
-
|
4
|
-
class Metacrunch::Elasticsearch::Cli::DumpMetadata < Metacrunch::Command
|
5
|
-
def call
|
6
|
-
[:mappings, :settings].each do |_element|
|
7
|
-
_filename_option = "#{_element}_filename"
|
8
|
-
|
9
|
-
if options[_filename_option].present?
|
10
|
-
options["dump_#{_element}"] = true
|
11
|
-
end
|
12
|
-
|
13
|
-
if options[_filename_option] == _filename_option
|
14
|
-
options.delete(_filename_option)
|
15
|
-
end
|
16
|
-
|
17
|
-
if options["dump_#{_element}"]
|
18
|
-
if filename = options["#{_element}_filename"]
|
19
|
-
File.write(filename, format(send(_element), format_from_filename(filename)))
|
20
|
-
else
|
21
|
-
puts send(_element)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
if !options[:dump_mapping] && !options[:dump_settings]
|
27
|
-
puts format(settings.merge(mappings), options[:format])
|
28
|
-
|
29
|
-
if options[:format] == :yaml
|
30
|
-
$stderr.puts <<-MESSAGE.strip_heredoc
|
31
|
-
|
32
|
-
You have requested to dump settings and mapping into YAML. Please keep in mind
|
33
|
-
that if you are trying to create an index with both, settings and mapping, this
|
34
|
-
request has to be JSON formatted. Nevertheless you can create the index with YAML
|
35
|
-
settings and put a YAML formatted mapping afterwards.
|
36
|
-
|
37
|
-
https://github.com/elastic/elasticsearch/issues/1755
|
38
|
-
|
39
|
-
MESSAGE
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
alias_method :perform, :call
|
44
|
-
|
45
|
-
private
|
46
|
-
|
47
|
-
def client
|
48
|
-
@client ||= Elasticsearch::Client.new(url: options[:url])
|
49
|
-
end
|
50
|
-
|
51
|
-
def format(obj, format)
|
52
|
-
if format.to_sym == :json
|
53
|
-
JSON.pretty_generate(obj)
|
54
|
-
elsif format.to_sym == :yaml
|
55
|
-
YAML.dump(obj)
|
56
|
-
else
|
57
|
-
raise "Unknown output format!"
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
def format_from_filename(filename)
|
62
|
-
return nil if filename.blank?
|
63
|
-
|
64
|
-
case File.extname(filename)
|
65
|
-
when /json\Z/i then :json
|
66
|
-
when /yml\Z|yaml\Z/i then :yaml
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
def mappings
|
71
|
-
client.indices.get_mapping(index: @options[:index], type: @options[:type]).try(:values).try(:first)
|
72
|
-
end
|
73
|
-
|
74
|
-
def settings
|
75
|
-
client.indices.get_settings(index: @options[:index])
|
76
|
-
.try(:values)
|
77
|
-
.try(:first)
|
78
|
-
.try(:tap) do |_obj|
|
79
|
-
_obj["settings"]["index"].reject! do |_key, _|
|
80
|
-
["creation_date", "uuid", "version"].include?(_key)
|
81
|
-
end
|
82
|
-
end
|
83
|
-
end
|
84
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
require "elasticsearch"
|
2
|
-
require_relative "../cli"
|
3
|
-
|
4
|
-
class Metacrunch::Elasticsearch::Cli::ListIndices < Metacrunch::Command
|
5
|
-
def call
|
6
|
-
puts indices
|
7
|
-
end
|
8
|
-
alias_method :perform, :call
|
9
|
-
|
10
|
-
private
|
11
|
-
|
12
|
-
def client
|
13
|
-
@client ||= Elasticsearch::Client.new(url: @options[:url])
|
14
|
-
end
|
15
|
-
|
16
|
-
def indices
|
17
|
-
client.cat.indices(h: "index", format: :json).map { |_element| _element["index"] }
|
18
|
-
end
|
19
|
-
end
|
data/lib/metacrunch_plugin.rb
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
require "metacrunch/elasticsearch"
|