elastic-util 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/elastic-util +9 -0
- data/lib/elastic_util.rb +9 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 101dd930f2901e53e2f0e3dd37b9a21ecee18265
|
4
|
+
data.tar.gz: ac928d8d5c722e947446b859f1ac10981783d1af
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d2eb13d33794f70d60a2dbec3186414ba5f5c2c51c0cebb7cf8b8220cd922102374de8e67c4e67bf6fb1738fb0dc1c36b6a80aaeff24c30a8d92fab70e4ca9e1
|
7
|
+
data.tar.gz: fbed8715e752f79c11b2eef136662ca43eef2bc5383cea70ea546877956a0f517a13a432b6c02e168c8c42399b624feaf3a34d04c1ff330616112933012b0421
|
data/bin/elastic-util
CHANGED
@@ -23,6 +23,15 @@ when "backup"
|
|
23
23
|
opts.on('--exclude-fields x,y,z', Array, "The fields to exclude from backup. Default is '_id'.") do |val|
|
24
24
|
options[:exclude_fields] = val.collect {|it| it.strip }
|
25
25
|
end
|
26
|
+
opts.on('--replace-types type1:_doc,type2:_doc', Array, "Replace certain types with a different type.") do |val|
|
27
|
+
options[:replace_types] = {}
|
28
|
+
val.each do |it|
|
29
|
+
pair = it.split(":").collect {|p| p.strip }
|
30
|
+
if pair.size == 2
|
31
|
+
options[:replace_types][pair[0]] = pair[1]
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
26
35
|
opts.on( '-s', '--size NUMBER', "The size api parameter. This dicates the size of the files and api payloads. Default is 1000." ) do |val|
|
27
36
|
options[:size] = val.to_i
|
28
37
|
end
|
data/lib/elastic_util.rb
CHANGED
@@ -12,7 +12,7 @@ require 'fileutils'
|
|
12
12
|
#
|
13
13
|
module ElasticUtil
|
14
14
|
|
15
|
-
VERSION = "0.1.
|
15
|
+
VERSION = "0.1.6"
|
16
16
|
|
17
17
|
# The name of the data directory, relative to the user provided backup directory.
|
18
18
|
DUMP_DIR = "es_data"
|
@@ -43,6 +43,7 @@ module ElasticUtil
|
|
43
43
|
# @option opts [Array] :indices The indices to backup. Default is all.
|
44
44
|
# @option opts [Array] :exclude_indices Exclude certain indexes.
|
45
45
|
# @option opts [Array] :exclude_fields Exclude certain fields. Default is ['_id'].
|
46
|
+
# @option opts [Array] :replace_types Replace certain types with a different type, separated by a colon. eg. 'type1:type2' or 'stat:_doc'
|
46
47
|
# @option opts [String] :scroll The scroll api parameter, Default is '5m'.
|
47
48
|
# @option opts [Integer] :size The size api parameter. Default is 1000.
|
48
49
|
# @option opts [true] :force Delete existing backup directory instead of erroring. Default is false.
|
@@ -212,7 +213,7 @@ module ElasticUtil
|
|
212
213
|
payload = File.read(file)
|
213
214
|
# uri = URI(url)
|
214
215
|
http = Net::HTTP.new(uri.host, uri.port)
|
215
|
-
response = http.post("/_bulk", payload)
|
216
|
+
response = http.post("/_bulk", payload, {"Content-Type" => "application/x-ndjson"})
|
216
217
|
if !response.is_a?(Net::HTTPSuccess)
|
217
218
|
raise Error, "HTTP request failure!\n#{response.inspect}\n#{response.body.to_s}"
|
218
219
|
end
|
@@ -231,8 +232,12 @@ module ElasticUtil
|
|
231
232
|
FileUtils.mkdir_p(dir_name)
|
232
233
|
file_name = File.join(dir_name, index_type) + (file_index ? "_#{file_index}" : "") + ".json.data"
|
233
234
|
# prepare record for bulk api injection
|
235
|
+
doc_type = hit['_type']
|
236
|
+
if opts[:replace_types] && opts[:replace_types][doc_type]
|
237
|
+
doc_type = opts[:replace_types][doc_type]
|
238
|
+
end
|
234
239
|
action_json = {'index' => {
|
235
|
-
'_index' => hit['_index'], '_type' =>
|
240
|
+
'_index' => hit['_index'], '_type' => doc_type, '_id' => hit['_id']
|
236
241
|
} }
|
237
242
|
source_json = hit['_source']
|
238
243
|
if opts[:exclude_fields] && source_json
|
@@ -240,6 +245,7 @@ module ElasticUtil
|
|
240
245
|
source_json.delete(field)
|
241
246
|
end
|
242
247
|
end
|
248
|
+
|
243
249
|
File.open(file_name, 'a') do |file|
|
244
250
|
file.write JSON.generate(action_json) + "\n" + JSON.generate(source_json) + "\n"
|
245
251
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elastic-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- James Dickson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-02-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|