ej 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +18 -0
- data/.pryrc +4 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +59 -0
- data/Rakefile +1 -0
- data/bin/ej +4 -0
- data/contrib/completion/_ej +3 -0
- data/ej.gemspec +30 -0
- data/lib/ej.rb +6 -0
- data/lib/ej/commands.rb +230 -0
- data/lib/ej/core.rb +227 -0
- data/lib/ej/version.rb +3 -0
- data/template/ejrc.erb +3 -0
- data/template/not_analyze_template.json +28 -0
- metadata +186 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 0ce4928c5040658e1671941f3a3e7091895098b4
|
4
|
+
data.tar.gz: c57ecc1f55a53acc4ea546244d5be96aea4d4ca5
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6705c528ab2e663560dc5d24e470fc0fa5396ef89840fe2cecb4c20b2aba69f7db3d33d33f518617018e6c52d95ae7a37cf40cf8a1500201280a4659987fc2d3
|
7
|
+
data.tar.gz: f36e354799a3b42370f45005ad2827dfa9f1f674ddfcc6ded150736a5086761ac5729b2aff28e2e1a049355c01f4fc1bdf59ccfb71a68702ac2ff3cd55ae807f
|
data/.gitignore
ADDED
data/.pryrc
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 toyama0919
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
# Ej
|
2
|
+
|
3
|
+
elasticsearch utility
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'ej'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install ej
|
18
|
+
|
19
|
+
|
20
|
+
## Usage
|
21
|
+
|
22
|
+
### simple search
|
23
|
+
```bash
|
24
|
+
ej -s
|
25
|
+
```
|
26
|
+
|
27
|
+
### other host
|
28
|
+
```bash
|
29
|
+
ej -s -h other_host
|
30
|
+
```
|
31
|
+
|
32
|
+
### query search and index
|
33
|
+
```bash
|
34
|
+
ej -s "ip_address: 127.0.0.1" -i logstash-2014.07.01
|
35
|
+
```
|
36
|
+
|
37
|
+
### index list
|
38
|
+
```bash
|
39
|
+
ej -l
|
40
|
+
```
|
41
|
+
|
42
|
+
### facet
|
43
|
+
```bash
|
44
|
+
ej -c ip_address -q "log_date: 2014-01-15"
|
45
|
+
```
|
46
|
+
|
47
|
+
### mapping
|
48
|
+
```bash
|
49
|
+
ej -m
|
50
|
+
```
|
51
|
+
|
52
|
+
|
53
|
+
## Contributing
|
54
|
+
|
55
|
+
1. Fork it ( http://github.com/toyama0919/ej/fork )
|
56
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
57
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
58
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
59
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/bin/ej
ADDED
data/ej.gemspec
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'ej/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "ej"
|
8
|
+
spec.version = Ej::VERSION
|
9
|
+
spec.authors = ["toyama0919"]
|
10
|
+
spec.email = ["toyama0919@gmail.com"]
|
11
|
+
spec.summary = %q{Command-line Elasticsearch TO JSON processor.}
|
12
|
+
spec.description = %q{Command-line Elasticsearch TO JSON processor.}
|
13
|
+
spec.homepage = ""
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.5"
|
22
|
+
spec.add_development_dependency "rake"
|
23
|
+
spec.add_development_dependency "pry"
|
24
|
+
spec.add_development_dependency "pry-doc"
|
25
|
+
spec.add_runtime_dependency "thor"
|
26
|
+
spec.add_runtime_dependency "yajl-ruby"
|
27
|
+
spec.add_runtime_dependency "elasticsearch"
|
28
|
+
spec.add_runtime_dependency "activesupport"
|
29
|
+
spec.add_runtime_dependency "hashie"
|
30
|
+
end
|
data/lib/ej.rb
ADDED
data/lib/ej/commands.rb
ADDED
@@ -0,0 +1,230 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# coding: utf-8
|
3
|
+
require 'thor'
|
4
|
+
require 'yajl'
|
5
|
+
require 'elasticsearch'
|
6
|
+
require 'ej/core'
|
7
|
+
require 'active_support/core_ext/array'
|
8
|
+
require 'active_support/core_ext/string'
|
9
|
+
require 'logger'
|
10
|
+
|
11
|
+
USER_SETTING_FILE = "#{ENV['HOME']}/.ejrc"
|
12
|
+
CURRENT_SETTING_FILE = ".ejrc"
|
13
|
+
|
14
|
+
module Ej
|
15
|
+
class Commands < Thor
|
16
|
+
class_option :index, aliases: '-i', type: :string, default: '_all', desc: 'index'
|
17
|
+
class_option :host, aliases: '-h', type: :string, default: 'localhost', desc: 'host'
|
18
|
+
class_option :profile, aliases: '-p', type: :string, default: 'default', desc: 'profile by .ejrc'
|
19
|
+
class_option :debug, aliases: '-d', type: :string, default: false, desc: 'debug mode'
|
20
|
+
map '-s' => :search
|
21
|
+
map '-c' => :facet
|
22
|
+
map '-I' => :bulk
|
23
|
+
map '-l' => :indices
|
24
|
+
map '-a' => :aliases
|
25
|
+
map '-m' => :mapping
|
26
|
+
map '-e' => :debug_eval
|
27
|
+
map '--j2h' => :json_to_hash
|
28
|
+
map '--health' => :health
|
29
|
+
|
30
|
+
def initialize(args = [], options = {}, config = {})
|
31
|
+
super(args, options, config)
|
32
|
+
@global_options = config[:shell].base.options
|
33
|
+
@core = Ej::Core.new(@global_options['host'], @global_options['index'], @global_options['debug'])
|
34
|
+
end
|
35
|
+
|
36
|
+
desc 'init', 'init'
|
37
|
+
def init
|
38
|
+
setting_file_path = "#{ENV['HOME']}/.ejrc"
|
39
|
+
default = {}
|
40
|
+
default['default'] = {}
|
41
|
+
default['default']['host'] = ask("What is default host?", default: 'localhost')
|
42
|
+
default['default']['port'] = ask("What is default port?", default: 9200)
|
43
|
+
default['default']['index'] = ask("What is default index?", default: '_all')
|
44
|
+
File.write(setting_file_path, default.to_yaml)
|
45
|
+
say("save setting file #{setting_file_path}", :green)
|
46
|
+
end
|
47
|
+
|
48
|
+
desc '-s [lucene query]', 'search'
|
49
|
+
option :type, type: :string, aliases: '-t', default: nil, desc: 'type'
|
50
|
+
option :size, type: :numeric, aliases: '-n', default: 10, desc: 'size'
|
51
|
+
option :from, type: :numeric, aliases: '--from', default: 0, desc: 'from'
|
52
|
+
option :source_only, type: :boolean, aliases: '--so', default: true, desc: 'from'
|
53
|
+
def search(query = nil)
|
54
|
+
puts_json(@core.search(options['type'], query, options['size'], options['from'], options['source_only']))
|
55
|
+
end
|
56
|
+
|
57
|
+
desc 'total [lucene query]', 'search'
|
58
|
+
option :type, type: :string, aliases: '-t', default: nil, desc: 'type'
|
59
|
+
def total(query = nil)
|
60
|
+
puts_json(@core.search(options['type'], query, 0, 0, false))
|
61
|
+
end
|
62
|
+
|
63
|
+
desc 'move', 'move index'
|
64
|
+
option :source, type: :string, aliases: '--source', required: true, desc: 'source host'
|
65
|
+
option :dest, type: :string, aliases: '--dest', required: true, desc: 'dest host'
|
66
|
+
option :query, type: :string, aliases: '-q', default: nil, desc: 'query'
|
67
|
+
def move
|
68
|
+
@core.move(options['source'], options['dest'], options['query'])
|
69
|
+
end
|
70
|
+
|
71
|
+
desc 'dump', 'move index'
|
72
|
+
option :query, type: :string, aliases: '-q', default: nil, desc: 'query'
|
73
|
+
def dump
|
74
|
+
@core.dump(options['query'])
|
75
|
+
end
|
76
|
+
|
77
|
+
desc '-c', 'facet'
|
78
|
+
option :query, type: :string, aliases: '-q', default: '*', desc: 'query'
|
79
|
+
option :size, type: :numeric, aliases: '-n', default: 10, desc: 'size'
|
80
|
+
def facet(term)
|
81
|
+
puts_json(@core.facet(term, options['size'], options['query']))
|
82
|
+
end
|
83
|
+
|
84
|
+
desc 'min', 'term'
|
85
|
+
option :term, type: :string, aliases: '-k', desc: 'terms'
|
86
|
+
def min
|
87
|
+
puts_json(@core.min(options['term']))
|
88
|
+
end
|
89
|
+
|
90
|
+
desc 'max', 'count record, group by keys'
|
91
|
+
option :term, type: :string, aliases: '-k', desc: 'terms'
|
92
|
+
def max
|
93
|
+
puts_json(@core.max(options['term']))
|
94
|
+
end
|
95
|
+
|
96
|
+
desc '-I', 'bulk import STDIN JSON'
|
97
|
+
option :index, aliases: '-i', type: :string, default: "logstash-#{Time.now.strftime('%Y.%m.%d')}", required: true, desc: 'index'
|
98
|
+
option :type, type: :string, aliases: '-t', default: nil, required: true, desc: 'type'
|
99
|
+
option :timestamp_key, aliases: '--timestamp_key', type: :string, desc: 'timestamp key', default: nil
|
100
|
+
option :add_timestamp, type: :boolean, default: true, desc: 'add_timestamp'
|
101
|
+
option :id_keys, type: :array, aliases: '--id', default: nil, desc: 'id'
|
102
|
+
def bulk
|
103
|
+
@core.bulk(options['timestamp_key'], options['type'], options['add_timestamp'], options['id_keys'], options['index'])
|
104
|
+
end
|
105
|
+
|
106
|
+
desc 'health', 'health'
|
107
|
+
def health
|
108
|
+
puts_json(@core.health)
|
109
|
+
end
|
110
|
+
|
111
|
+
desc '-a', 'list aliases'
|
112
|
+
def aliases
|
113
|
+
puts_json(@core.aliases)
|
114
|
+
end
|
115
|
+
|
116
|
+
desc 'state', 'health'
|
117
|
+
def state
|
118
|
+
puts_json(@core.state)
|
119
|
+
end
|
120
|
+
|
121
|
+
desc 'indices', 'indices'
|
122
|
+
def indices
|
123
|
+
puts_json(@core.indices)
|
124
|
+
end
|
125
|
+
|
126
|
+
desc 'count', 'count'
|
127
|
+
def count
|
128
|
+
puts_json(@core.count)
|
129
|
+
end
|
130
|
+
|
131
|
+
desc 'stats', 'count'
|
132
|
+
def stats
|
133
|
+
puts_json(@core.stats)
|
134
|
+
end
|
135
|
+
|
136
|
+
desc 'mapping', 'count'
|
137
|
+
def mapping
|
138
|
+
puts_json(@core.mapping)
|
139
|
+
end
|
140
|
+
|
141
|
+
desc 'not_analyzed', 'not_analyzed'
|
142
|
+
def not_analyzed
|
143
|
+
json = File.read(File.expand_path('../../../template/not_analyze_template.json', __FILE__))
|
144
|
+
hash = Yajl::Parser.parse(json)
|
145
|
+
puts_json(@core.put_template('ej_init', hash))
|
146
|
+
end
|
147
|
+
|
148
|
+
desc 'put_routing', 'put routing'
|
149
|
+
option :index, aliases: '-i', type: :string, default: nil, required: true, desc: 'index'
|
150
|
+
option :type, aliases: '-t', type: :string, default: nil, required: true, desc: 'type'
|
151
|
+
option :path, type: :string, default: nil, required: true, desc: 'path'
|
152
|
+
def put_routing
|
153
|
+
body = { options['type'] => {"_routing"=>{"required"=>true, "path"=>options['path']}}}
|
154
|
+
puts_json(@core.put_mapping(options['index'], options['type'], body))
|
155
|
+
end
|
156
|
+
|
157
|
+
desc 'put_template', 'put_template'
|
158
|
+
def put_template(name)
|
159
|
+
hash = Yajl::Parser.parse(STDIN.read)
|
160
|
+
puts_json(@core.put_template(name, hash))
|
161
|
+
end
|
162
|
+
|
163
|
+
desc 'create_aliases', 'create_aliases'
|
164
|
+
option :alias, type: :string, aliases: '-a', default: nil, required: true, desc: 'type'
|
165
|
+
option :indices, type: :array, aliases: '-x', default: nil, required: true, desc: 'type'
|
166
|
+
def create_aliases
|
167
|
+
@core.create_aliases(options['alias'], options['indices'])
|
168
|
+
end
|
169
|
+
|
170
|
+
desc 'recovery', 'recovery'
|
171
|
+
def recovery
|
172
|
+
@core.recovery
|
173
|
+
end
|
174
|
+
|
175
|
+
desc 'delete', 'delete index'
|
176
|
+
option :index, aliases: '-i', type: :string, default: nil, required: true, desc: 'profile by .database.yml'
|
177
|
+
option :query, type: :string, aliases: '-q', default: nil, desc: 'query'
|
178
|
+
def delete
|
179
|
+
@core.delete(options['index'], options['query'])
|
180
|
+
end
|
181
|
+
|
182
|
+
desc 'delete_template --name [name]', 'delete_template'
|
183
|
+
option :name, type: :string, default: nil, required: true, desc: 'template name'
|
184
|
+
def delete_template
|
185
|
+
@core.delete_template(options['name'])
|
186
|
+
end
|
187
|
+
|
188
|
+
desc 'template', 'get template'
|
189
|
+
def template
|
190
|
+
puts_json(@core.template)
|
191
|
+
end
|
192
|
+
|
193
|
+
desc 'settings', 'get template'
|
194
|
+
def settings
|
195
|
+
puts_json(@core.settings)
|
196
|
+
end
|
197
|
+
|
198
|
+
desc 'warmer', 'get warmer'
|
199
|
+
def warmer
|
200
|
+
puts_json(@core.warmer)
|
201
|
+
end
|
202
|
+
|
203
|
+
desc 'refresh', 'get refresh'
|
204
|
+
def refresh
|
205
|
+
puts_json(@core.refresh)
|
206
|
+
end
|
207
|
+
|
208
|
+
desc '--j2h', 'json to hash'
|
209
|
+
def json_to_hash
|
210
|
+
pp Yajl::Parser.parse(STDIN.read)
|
211
|
+
end
|
212
|
+
|
213
|
+
private
|
214
|
+
|
215
|
+
def puts_json(object)
|
216
|
+
puts Yajl::Encoder.encode(object)
|
217
|
+
end
|
218
|
+
|
219
|
+
def setting
|
220
|
+
if File.exist?(CURRENT_SETTING_FILE)
|
221
|
+
return YAML.load_file(CURRENT_SETTING_FILE)
|
222
|
+
end
|
223
|
+
|
224
|
+
if File.exist?(USER_SETTING_FILE)
|
225
|
+
return YAML.load_file(USER_SETTING_FILE)
|
226
|
+
end
|
227
|
+
return { host: 'localhost', index: '_all', port: 9200 }
|
228
|
+
end
|
229
|
+
end
|
230
|
+
end
|
data/lib/ej/core.rb
ADDED
@@ -0,0 +1,227 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# coding: utf-8
|
3
|
+
require 'yaml'
|
4
|
+
require 'yajl'
|
5
|
+
require 'elasticsearch'
|
6
|
+
require 'hashie'
|
7
|
+
require 'pp'
|
8
|
+
|
9
|
+
module Ej
|
10
|
+
class Core
|
11
|
+
def initialize(host, index, debug)
|
12
|
+
@logger = Logger.new($stderr)
|
13
|
+
@logger.level = debug ? Logger::DEBUG : Logger::INFO
|
14
|
+
@index = index
|
15
|
+
@client = Elasticsearch::Client.new hosts: host, logger: @logger, index: @index
|
16
|
+
end
|
17
|
+
|
18
|
+
def search(type, query, size, from, source_only, routing = nil)
|
19
|
+
body = { size: size, from: from }
|
20
|
+
body[:query] = { query_string: { query: query } } unless query.nil?
|
21
|
+
search_option = { index: @index, type: type, body: body }
|
22
|
+
search_option[:routing] = routing unless routing.nil?
|
23
|
+
results = Hashie::Mash.new(@client.search(search_option))
|
24
|
+
source_only ? get_sources(results) : results
|
25
|
+
end
|
26
|
+
|
27
|
+
def move(source, dest, query)
|
28
|
+
per = 30000
|
29
|
+
source_client = Elasticsearch::Client.new hosts: source, index: @index, logger: @logger
|
30
|
+
dest_client = Elasticsearch::Client.new hosts: dest, logger: @logger
|
31
|
+
num = 0
|
32
|
+
while true
|
33
|
+
from = num * per
|
34
|
+
body = { size: per, from: from }
|
35
|
+
body[:query] = { query_string: { query: query } } unless query.nil?
|
36
|
+
data = Hashie::Mash.new(source_client.search index: @index, body: body)
|
37
|
+
break if data.hits.hits.empty?
|
38
|
+
bulk_message = []
|
39
|
+
data.hits.hits.each do |doc|
|
40
|
+
bulk_message << { 'index' => { '_index' => doc._index, '_type' => doc._type, '_id' => doc._id } }
|
41
|
+
bulk_message << doc._source
|
42
|
+
end
|
43
|
+
dest_client.bulk body: bulk_message unless bulk_message.empty?
|
44
|
+
num += 1
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def dump(query)
|
49
|
+
per = 30000
|
50
|
+
num = 0
|
51
|
+
bulk_message = []
|
52
|
+
while true
|
53
|
+
from = num * per
|
54
|
+
body = { size: per, from: from }
|
55
|
+
body[:query] = { query_string: { query: query } } unless query.nil?
|
56
|
+
data = Hashie::Mash.new(@client.search index: @index, body: body)
|
57
|
+
break if data.hits.hits.empty?
|
58
|
+
data.hits.hits.each do |doc|
|
59
|
+
source = doc.delete('_source')
|
60
|
+
doc.delete('_score')
|
61
|
+
bulk_message << Yajl::Encoder.encode({ 'index' => doc.to_h })
|
62
|
+
bulk_message << Yajl::Encoder.encode(source)
|
63
|
+
end
|
64
|
+
num += 1
|
65
|
+
end
|
66
|
+
puts bulk_message.join("\n")
|
67
|
+
end
|
68
|
+
|
69
|
+
def facet(term, size, query)
|
70
|
+
body = {"facets"=>
|
71
|
+
{"terms"=>
|
72
|
+
{"terms"=>{"field"=>term, "size"=>size, "order"=>"count", "exclude"=>[]},
|
73
|
+
"facet_filter"=>
|
74
|
+
{"fquery"=>
|
75
|
+
{"query"=>
|
76
|
+
{"filtered"=>
|
77
|
+
{"query"=>
|
78
|
+
{"bool"=>
|
79
|
+
{"should"=>[{"query_string"=>{"query"=>query}}]}},
|
80
|
+
"filter"=>{"bool"=>{"must"=>[{"match_all"=>{}}]}}}}}}}},
|
81
|
+
"size"=>0}
|
82
|
+
@client.search index: @index, body: body
|
83
|
+
end
|
84
|
+
|
85
|
+
def min(term)
|
86
|
+
body = {
|
87
|
+
aggs: {
|
88
|
+
"min_#{term}" => { min: { field: term } }
|
89
|
+
}
|
90
|
+
}
|
91
|
+
@client.search index: @index, body: body, size: 0
|
92
|
+
end
|
93
|
+
|
94
|
+
def max(term)
|
95
|
+
body = {
|
96
|
+
aggs: {
|
97
|
+
"max_#{term}" => { max: { field: term } }
|
98
|
+
}
|
99
|
+
}
|
100
|
+
@client.search index: @index, body: body, size: 0
|
101
|
+
end
|
102
|
+
|
103
|
+
def aliases
|
104
|
+
@client.indices.get_aliases
|
105
|
+
end
|
106
|
+
|
107
|
+
def health
|
108
|
+
@client.cluster.health
|
109
|
+
end
|
110
|
+
|
111
|
+
def state
|
112
|
+
@client.cluster.state
|
113
|
+
end
|
114
|
+
|
115
|
+
def indices
|
116
|
+
@client.cat.indices format: 'json'
|
117
|
+
end
|
118
|
+
|
119
|
+
def count
|
120
|
+
@client.cat.count index: @index, format: 'json'
|
121
|
+
end
|
122
|
+
|
123
|
+
def stats
|
124
|
+
@client.indices.stats index: @index
|
125
|
+
end
|
126
|
+
|
127
|
+
def put_mapping(index, type, body)
|
128
|
+
@client.indices.create index: index unless @client.indices.exists index: index
|
129
|
+
@client.indices.put_mapping index: index, type: type, body: body
|
130
|
+
end
|
131
|
+
|
132
|
+
def mapping
|
133
|
+
data = @client.indices.get_mapping index: @index
|
134
|
+
@index == '_all' ? data : data[@index]['mappings']
|
135
|
+
end
|
136
|
+
|
137
|
+
def put_template(name, hash)
|
138
|
+
@client.indices.put_template name: name, body: hash
|
139
|
+
end
|
140
|
+
|
141
|
+
def create_aliases(als, indices)
|
142
|
+
actions = []
|
143
|
+
indices.each do |index|
|
144
|
+
actions << { add: { index: index, alias: als } }
|
145
|
+
end
|
146
|
+
@client.indices.update_aliases body: {
|
147
|
+
actions: actions
|
148
|
+
}
|
149
|
+
end
|
150
|
+
|
151
|
+
def recovery
|
152
|
+
@client.indices.recovery index: @index
|
153
|
+
end
|
154
|
+
|
155
|
+
def delete(index, query)
|
156
|
+
if query.nil?
|
157
|
+
@client.indices.delete index: index
|
158
|
+
else
|
159
|
+
@client.delete_by_query index: index, q: query
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
def template
|
164
|
+
@client.indices.get_template
|
165
|
+
end
|
166
|
+
|
167
|
+
def delete_template(name)
|
168
|
+
@client.indices.delete_template name: name
|
169
|
+
end
|
170
|
+
|
171
|
+
def settings
|
172
|
+
@client.indices.get_settings
|
173
|
+
end
|
174
|
+
|
175
|
+
def warmer
|
176
|
+
@client.indices.get_warmer index: @index
|
177
|
+
end
|
178
|
+
|
179
|
+
def refresh
|
180
|
+
@client.indices.refresh index: @index
|
181
|
+
end
|
182
|
+
|
183
|
+
def bulk(timestamp_key, type, add_timestamp, id_keys, index)
|
184
|
+
data = parse_json(STDIN.read)
|
185
|
+
template = id_keys.map { |key| '%s' }.join('_') unless id_keys.nil?
|
186
|
+
bulk_message = []
|
187
|
+
data.each do |record|
|
188
|
+
if timestamp_key.nil?
|
189
|
+
timestamp = Time.now.to_datetime.to_s
|
190
|
+
else
|
191
|
+
timestamp = record[timestamp_key].to_time.to_datetime.to_s
|
192
|
+
end
|
193
|
+
record.merge!('@timestamp' => timestamp) if add_timestamp
|
194
|
+
meta = { index: { _index: index, _type: type } }
|
195
|
+
meta[:index][:_id] = generate_id(template, record, id_keys) unless id_keys.nil?
|
196
|
+
bulk_message << meta
|
197
|
+
bulk_message << record
|
198
|
+
end
|
199
|
+
bulk_message.in_groups_of(10000, false) do |block|
|
200
|
+
@client.bulk body: block
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
private
|
205
|
+
|
206
|
+
def parse_json(buffer)
|
207
|
+
begin
|
208
|
+
data = Yajl::Parser.parse(buffer)
|
209
|
+
rescue => e
|
210
|
+
data = []
|
211
|
+
buffer.split("\n").each do |line|
|
212
|
+
data << Yajl::Parser.parse(line)
|
213
|
+
end
|
214
|
+
end
|
215
|
+
data.class == Array ? data : [data]
|
216
|
+
end
|
217
|
+
|
218
|
+
def generate_id(template, record, id_keys)
|
219
|
+
template % id_keys.map { |key| record[key] }
|
220
|
+
end
|
221
|
+
|
222
|
+
def get_sources(results)
|
223
|
+
results.hits.hits.map { |result| result._source }
|
224
|
+
end
|
225
|
+
|
226
|
+
end
|
227
|
+
end
|
data/lib/ej/version.rb
ADDED
data/template/ejrc.erb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
{
|
2
|
+
"template": "*",
|
3
|
+
"settings" : {
|
4
|
+
"number_of_shards" : 5,
|
5
|
+
"number_of_replicas" : 0,
|
6
|
+
"index" : {
|
7
|
+
"store" : { "compress" : { "stored" : true, "tv": true } }
|
8
|
+
}
|
9
|
+
},
|
10
|
+
"mappings": {
|
11
|
+
"_default_": {
|
12
|
+
"_all": { "enabled": false },
|
13
|
+
"_source": { "compress": true },
|
14
|
+
"dynamic_templates": [
|
15
|
+
{
|
16
|
+
"string_template" : {
|
17
|
+
"match" : "*",
|
18
|
+
"mapping": { "type": "string", "index": "not_analyzed" },
|
19
|
+
"match_mapping_type" : "string"
|
20
|
+
}
|
21
|
+
}
|
22
|
+
],
|
23
|
+
"properties" : {
|
24
|
+
"@timestamp" : { "type" : "date", "index" : "not_analyzed" }
|
25
|
+
}
|
26
|
+
}
|
27
|
+
}
|
28
|
+
}
|
metadata
ADDED
@@ -0,0 +1,186 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ej
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.4
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- toyama0919
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-08-07 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.5'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.5'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: pry
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: pry-doc
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: thor
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: yajl-ruby
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: elasticsearch
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: activesupport
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: hashie
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
description: Command-line Elasticsearch TO JSON processor.
|
140
|
+
email:
|
141
|
+
- toyama0919@gmail.com
|
142
|
+
executables:
|
143
|
+
- ej
|
144
|
+
extensions: []
|
145
|
+
extra_rdoc_files: []
|
146
|
+
files:
|
147
|
+
- ".gitignore"
|
148
|
+
- ".pryrc"
|
149
|
+
- Gemfile
|
150
|
+
- LICENSE.txt
|
151
|
+
- README.md
|
152
|
+
- Rakefile
|
153
|
+
- bin/ej
|
154
|
+
- contrib/completion/_ej
|
155
|
+
- ej.gemspec
|
156
|
+
- lib/ej.rb
|
157
|
+
- lib/ej/commands.rb
|
158
|
+
- lib/ej/core.rb
|
159
|
+
- lib/ej/version.rb
|
160
|
+
- template/ejrc.erb
|
161
|
+
- template/not_analyze_template.json
|
162
|
+
homepage: ''
|
163
|
+
licenses:
|
164
|
+
- MIT
|
165
|
+
metadata: {}
|
166
|
+
post_install_message:
|
167
|
+
rdoc_options: []
|
168
|
+
require_paths:
|
169
|
+
- lib
|
170
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
171
|
+
requirements:
|
172
|
+
- - ">="
|
173
|
+
- !ruby/object:Gem::Version
|
174
|
+
version: '0'
|
175
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
176
|
+
requirements:
|
177
|
+
- - ">="
|
178
|
+
- !ruby/object:Gem::Version
|
179
|
+
version: '0'
|
180
|
+
requirements: []
|
181
|
+
rubyforge_project:
|
182
|
+
rubygems_version: 2.2.2
|
183
|
+
signing_key:
|
184
|
+
specification_version: 4
|
185
|
+
summary: Command-line Elasticsearch TO JSON processor.
|
186
|
+
test_files: []
|