elastic-util 0.1.6 → 0.1.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/lib/elastic_util.rb +59 -23
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: af7028b9a21af54dc8f66849ce7a22e9739dbf0a2b7d8d2fd4091144337b3fda
|
4
|
+
data.tar.gz: 8caf084b468de28607516b1c8d8643de9373732a5d1628db4618307f6b7dac64
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 07ce1c430cd062388456fc8d19717546457aba8bd20952a54697d17192a36e63367d66a37f78f8fc08d48d0feb89fd5c26c53fccc32a0f5df26e65e31278bc73
|
7
|
+
data.tar.gz: bb50a5ada50644ee7edaf831af2d1966d4610515d268310084a01c0d0f278c2679c3a6757fc62ee6c6863c5f53c03842a2190335ed807037c97d7cd2eb004a38
|
data/lib/elastic_util.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'net/http'
|
3
|
+
require 'openssl'
|
3
4
|
require 'json'
|
4
5
|
require 'fileutils'
|
5
6
|
|
@@ -12,7 +13,7 @@ require 'fileutils'
|
|
12
13
|
#
|
13
14
|
module ElasticUtil
|
14
15
|
|
15
|
-
VERSION = "0.1.
|
16
|
+
VERSION = "0.1.7"
|
16
17
|
|
17
18
|
# The name of the data directory, relative to the user provided backup directory.
|
18
19
|
DUMP_DIR = "es_data"
|
@@ -53,19 +54,14 @@ module ElasticUtil
|
|
53
54
|
#
|
54
55
|
def self.backup(url, backup_dir, opts={})
|
55
56
|
start_time = Time.now
|
57
|
+
url = url.strip.chomp("/")
|
56
58
|
backup_dir = backup_dir.strip
|
57
59
|
path = File.join(backup_dir.strip, DUMP_DIR)
|
58
60
|
indices = []
|
59
|
-
|
60
61
|
# ping it first
|
62
|
+
response = nil
|
61
63
|
uri = URI(url)
|
62
|
-
response =
|
63
|
-
http = Net::HTTP.new(uri.host, uri.port)
|
64
|
-
http.read_timeout = (60*3)
|
65
|
-
http.open_timeout = 5
|
66
|
-
response = http.start() {|http|
|
67
|
-
http.get("/")
|
68
|
-
}
|
64
|
+
response = api_get(uri)
|
69
65
|
if !response.is_a?(Net::HTTPSuccess)
|
70
66
|
raise Error, "Unable to reach elasticsearch at url '#{url}'!\n#{response.inspect}\n#{response.body.to_s}"
|
71
67
|
end
|
@@ -74,8 +70,9 @@ module ElasticUtil
|
|
74
70
|
if opts[:indices]
|
75
71
|
indices = opts[:indices]
|
76
72
|
else
|
73
|
+
response = nil
|
77
74
|
uri = URI(url + "/_cat/indices?format=json")
|
78
|
-
response =
|
75
|
+
response = api_get(uri)
|
79
76
|
if !response.is_a?(Net::HTTPSuccess)
|
80
77
|
raise Error, "HTTP request failure!\n#{response.inspect}\n#{response.body.to_s}"
|
81
78
|
end
|
@@ -124,7 +121,7 @@ module ElasticUtil
|
|
124
121
|
}
|
125
122
|
uri.query = URI.encode_www_form(params)
|
126
123
|
# puts "HTTP REQUEST #{uri.inspect}"
|
127
|
-
response =
|
124
|
+
response = api_get(uri)
|
128
125
|
if !response.is_a?(Net::HTTPSuccess)
|
129
126
|
raise Error, "HTTP request failure!\n#{response.inspect}\n#{response.body.to_s}"
|
130
127
|
end
|
@@ -144,7 +141,7 @@ module ElasticUtil
|
|
144
141
|
}
|
145
142
|
uri.query = URI.encode_www_form(params)
|
146
143
|
# puts "HTTP REQUEST #{uri.inspect}"
|
147
|
-
response =
|
144
|
+
response = api_get(uri)
|
148
145
|
if !response.is_a?(Net::HTTPSuccess)
|
149
146
|
raise Error, "HTTP request failure!\n#{response.inspect}\n#{response.body.to_s}"
|
150
147
|
end
|
@@ -187,14 +184,7 @@ module ElasticUtil
|
|
187
184
|
|
188
185
|
# ping it first
|
189
186
|
uri = URI(url)
|
190
|
-
response =
|
191
|
-
http = Net::HTTP.new(uri.host, uri.port)
|
192
|
-
http.read_timeout = (60*3)
|
193
|
-
http.open_timeout = 5
|
194
|
-
response = http.start() {|http|
|
195
|
-
http.get("/")
|
196
|
-
}
|
197
|
-
|
187
|
+
response = api_get(uri)
|
198
188
|
if !response.is_a?(Net::HTTPSuccess)
|
199
189
|
raise Error, "Unable to reach elasticsearch at url '#{url}'!\n#{response.inspect}\n#{response.body.to_s}"
|
200
190
|
end
|
@@ -211,9 +201,8 @@ module ElasticUtil
|
|
211
201
|
found_files.each_with_index do |file, i|
|
212
202
|
puts "(#{i+1}/#{found_files.size}) bulk importing file #{file}" unless opts[:quiet]
|
213
203
|
payload = File.read(file)
|
214
|
-
|
215
|
-
|
216
|
-
response = http.post("/_bulk", payload, {"Content-Type" => "application/x-ndjson"})
|
204
|
+
uri = URI(url + "/_bulk")
|
205
|
+
response = api_post(uri, payload, {:headers => {"Content-Type" => "application/x-ndjson"} })
|
217
206
|
if !response.is_a?(Net::HTTPSuccess)
|
218
207
|
raise Error, "HTTP request failure!\n#{response.inspect}\n#{response.body.to_s}"
|
219
208
|
end
|
@@ -253,4 +242,51 @@ module ElasticUtil
|
|
253
242
|
end
|
254
243
|
end
|
255
244
|
|
245
|
+
def self.api_get(uri, opts={})
|
246
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
247
|
+
if uri.scheme == 'https'
|
248
|
+
http.use_ssl = true
|
249
|
+
# todo: always ignore ssl errors for now, but this should be an option
|
250
|
+
# http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
251
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
252
|
+
end
|
253
|
+
http.read_timeout = opts[:read_timeout] || (60*15)
|
254
|
+
http.open_timeout = opts[:open_timeout] || 5
|
255
|
+
request = Net::HTTP::Get.new uri.request_uri
|
256
|
+
if opts[:headers]
|
257
|
+
opts[:headers].each do |k,v|
|
258
|
+
request[k] = v
|
259
|
+
end
|
260
|
+
end
|
261
|
+
if uri.user
|
262
|
+
request.basic_auth uri.user, uri.password
|
263
|
+
end
|
264
|
+
response = http.request(request)
|
265
|
+
return response
|
266
|
+
end
|
267
|
+
|
268
|
+
def self.api_post(uri, payload, opts={})
|
269
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
270
|
+
if uri.scheme == 'https'
|
271
|
+
http.use_ssl = true
|
272
|
+
# todo: always ignore ssl errors for now, but this should be an option
|
273
|
+
# http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
274
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
275
|
+
end
|
276
|
+
http.read_timeout = opts[:read_timeout] || (60*15)
|
277
|
+
http.open_timeout = opts[:open_timeout] || 5
|
278
|
+
request = Net::HTTP::Post.new uri.request_uri
|
279
|
+
if opts[:headers]
|
280
|
+
opts[:headers].each do |k,v|
|
281
|
+
request[k] = v
|
282
|
+
end
|
283
|
+
end
|
284
|
+
if uri.user
|
285
|
+
request.basic_auth uri.user, uri.password
|
286
|
+
end
|
287
|
+
request.body = payload
|
288
|
+
response = http.request(request)
|
289
|
+
return response
|
290
|
+
end
|
291
|
+
|
256
292
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elastic-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- James Dickson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-06-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -75,7 +75,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
75
75
|
version: '0'
|
76
76
|
requirements: []
|
77
77
|
rubyforge_project:
|
78
|
-
rubygems_version: 2.
|
78
|
+
rubygems_version: 2.7.6
|
79
79
|
signing_key:
|
80
80
|
specification_version: 4
|
81
81
|
summary: Provides backup and restore for ElasticSearch data
|