elastic-util 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 631179e3ea01a206b5ddfe51ac9189a55b4e9988
4
- data.tar.gz: 245f484434a40d4a86057e519a3ff93a508160ea
3
+ metadata.gz: 101dd930f2901e53e2f0e3dd37b9a21ecee18265
4
+ data.tar.gz: ac928d8d5c722e947446b859f1ac10981783d1af
5
5
  SHA512:
6
- metadata.gz: c56b125c357a4101a9ee8514435638f3c231c6a10fe2130cb49cd1a2a7ad46229b0ea0d023d92c91915f970b60c2907145f3c4c0ff4d4fc50e824e9839ffe3d4
7
- data.tar.gz: db2d9d011bc14e7f6e3c64b21a9a31e30d8844541534a3cdffef8cfd67c799c37b1d42bfb2893c0491734e6cf1e8b8338e04264c0e337cea0bb38c01ba0c0929
6
+ metadata.gz: d2eb13d33794f70d60a2dbec3186414ba5f5c2c51c0cebb7cf8b8220cd922102374de8e67c4e67bf6fb1738fb0dc1c36b6a80aaeff24c30a8d92fab70e4ca9e1
7
+ data.tar.gz: fbed8715e752f79c11b2eef136662ca43eef2bc5383cea70ea546877956a0f517a13a432b6c02e168c8c42399b624feaf3a34d04c1ff330616112933012b0421
@@ -23,6 +23,15 @@ when "backup"
23
23
  opts.on('--exclude-fields x,y,z', Array, "The fields to exclude from backup. Default is '_id'.") do |val|
24
24
  options[:exclude_fields] = val.collect {|it| it.strip }
25
25
  end
26
+ opts.on('--replace-types type1:_doc,type2:_doc', Array, "Replace certain types with a different type.") do |val|
27
+ options[:replace_types] = {}
28
+ val.each do |it|
29
+ pair = it.split(":").collect {|p| p.strip }
30
+ if pair.size == 2
31
+ options[:replace_types][pair[0]] = pair[1]
32
+ end
33
+ end
34
+ end
26
35
  opts.on( '-s', '--size NUMBER', "The size api parameter. This dicates the size of the files and api payloads. Default is 1000." ) do |val|
27
36
  options[:size] = val.to_i
28
37
  end
@@ -12,7 +12,7 @@ require 'fileutils'
12
12
  #
13
13
  module ElasticUtil
14
14
 
15
- VERSION = "0.1.5"
15
+ VERSION = "0.1.6"
16
16
 
17
17
  # The name of the data directory, relative to the user provided backup directory.
18
18
  DUMP_DIR = "es_data"
@@ -43,6 +43,7 @@ module ElasticUtil
43
43
  # @option opts [Array] :indices The indices to backup. Default is all.
44
44
  # @option opts [Array] :exclude_indices Exclude certain indexes.
45
45
  # @option opts [Array] :exclude_fields Exclude certain fields. Default is ['_id'].
46
+ # @option opts [Array] :replace_types Replace certain types with a different type, separated by a colon. eg. 'type1:type2' or 'stat:_doc'
46
47
  # @option opts [String] :scroll The scroll api parameter, Default is '5m'.
47
48
  # @option opts [Integer] :size The size api parameter. Default is 1000.
48
49
  # @option opts [true] :force Delete existing backup directory instead of erroring. Default is false.
@@ -212,7 +213,7 @@ module ElasticUtil
212
213
  payload = File.read(file)
213
214
  # uri = URI(url)
214
215
  http = Net::HTTP.new(uri.host, uri.port)
215
- response = http.post("/_bulk", payload)
216
+ response = http.post("/_bulk", payload, {"Content-Type" => "application/x-ndjson"})
216
217
  if !response.is_a?(Net::HTTPSuccess)
217
218
  raise Error, "HTTP request failure!\n#{response.inspect}\n#{response.body.to_s}"
218
219
  end
@@ -231,8 +232,12 @@ module ElasticUtil
231
232
  FileUtils.mkdir_p(dir_name)
232
233
  file_name = File.join(dir_name, index_type) + (file_index ? "_#{file_index}" : "") + ".json.data"
233
234
  # prepare record for bulk api injection
235
+ doc_type = hit['_type']
236
+ if opts[:replace_types] && opts[:replace_types][doc_type]
237
+ doc_type = opts[:replace_types][doc_type]
238
+ end
234
239
  action_json = {'index' => {
235
- '_index' => hit['_index'], '_type' => hit['_type'], '_id' => hit['_id']
240
+ '_index' => hit['_index'], '_type' => doc_type, '_id' => hit['_id']
236
241
  } }
237
242
  source_json = hit['_source']
238
243
  if opts[:exclude_fields] && source_json
@@ -240,6 +245,7 @@ module ElasticUtil
240
245
  source_json.delete(field)
241
246
  end
242
247
  end
248
+
243
249
  File.open(file_name, 'a') do |file|
244
250
  file.write JSON.generate(action_json) + "\n" + JSON.generate(source_json) + "\n"
245
251
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elastic-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 0.1.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - James Dickson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-06-12 00:00:00.000000000 Z
11
+ date: 2020-02-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler