secateurs 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +2 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +263 -0
- data/Rakefile +6 -0
- data/bin/secateurs +5 -0
- data/examples/Templatefile +5 -0
- data/examples/kafkabeat.rb +84 -0
- data/examples/template_1.rb +33 -0
- data/examples/templates/template_bar.rb +24 -0
- data/examples/templates/template_foo.rb +33 -0
- data/lib/secateurs.rb +15 -0
- data/lib/secateurs/cli.rb +36 -0
- data/lib/secateurs/client.rb +25 -0
- data/lib/secateurs/comparer.rb +35 -0
- data/lib/secateurs/dsl.rb +59 -0
- data/lib/secateurs/dsl/converter.rb +75 -0
- data/lib/secateurs/dsl/template_builder.rb +11 -0
- data/lib/secateurs/exporter.rb +26 -0
- data/lib/secateurs/exporter/base.rb +42 -0
- data/lib/secateurs/exporter/json_exporter.rb +9 -0
- data/lib/secateurs/exporter/ruby_exporter.rb +22 -0
- data/lib/secateurs/exporter/split_exporter.rb +39 -0
- data/lib/secateurs/exporter/yaml_exporter.rb +9 -0
- data/lib/secateurs/parser.rb +26 -0
- data/lib/secateurs/runner.rb +44 -0
- data/lib/secateurs/version.rb +3 -0
- data/secateurs.gemspec +31 -0
- data/wercker.yml +25 -0
- metadata +200 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: beccaf2f0361cabeea578a164a7c49ddb12da43b
|
4
|
+
data.tar.gz: d9ac2bb185532038744adebd0ed85c43935f6328
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 2246d9c84aac58d8173f8675d558043350a29c591704e1e192b749913ddae9189714ff60a0552723166a1cd091ce2c63c3aeb4b08c55b9c022bb85fb39c3bd6c
|
7
|
+
data.tar.gz: 586f5c510c2294556597fef41a46802076bcfe348abe48b5e745f212048e347eb05da60a58a9cc14bd74178d959a286413956253b0b12ee14eb33736e98e5e26
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2016 Daichi Hirata
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,263 @@
|
|
1
|
+
# Secateurs
|
2
|
+
|
3
|
+
Secateurs is a tool to manage Elasticsearch Index Template.
|
4
|
+
|
5
|
+
[](https://app.wercker.com/project/bykey/7090ad6286de2e46db8d6d84b3242a09) [](https://codeclimate.com/github/daichirata/secateurs)
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'secateurs'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install secateurs
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
### Commands
|
26
|
+
|
27
|
+
```sh
|
28
|
+
$ secateurs help
|
29
|
+
Commands:
|
30
|
+
secateurs apply FILE # apply index template file.
|
31
|
+
secateurs export # export index template to local file.
|
32
|
+
secateurs help [COMMAND] # Describe available commands or one specific command
|
33
|
+
```
|
34
|
+
|
35
|
+
### Apply
|
36
|
+
|
37
|
+
```sh
|
38
|
+
$ secateurs help apply
|
39
|
+
Usage:
|
40
|
+
secateurs apply FILE
|
41
|
+
|
42
|
+
Options:
|
43
|
+
-h, [--host=HOST]
|
44
|
+
# Default: localhost
|
45
|
+
-p, [--port=N]
|
46
|
+
# Default: 9200
|
47
|
+
-f, [--format=FORMAT]
|
48
|
+
# Default: ruby
|
49
|
+
# Possible values: ruby, json, yaml
|
50
|
+
[--color], [--no-color]
|
51
|
+
# Default: true
|
52
|
+
[--dry-run], [--no-dry-run]
|
53
|
+
[--verbose], [--no-verbose]
|
54
|
+
|
55
|
+
apply index template file.
|
56
|
+
```
|
57
|
+
|
58
|
+
```sh
|
59
|
+
$ secateurs apply Templatefile
|
60
|
+
|
61
|
+
$ secateurs apply --dry-run Templatefile.json
|
62
|
+
|
63
|
+
$ secateurs apply --format json Templatefile.json
|
64
|
+
```
|
65
|
+
|
66
|
+
### Export
|
67
|
+
|
68
|
+
```sh
|
69
|
+
$ secateurs help export
|
70
|
+
Usage:
|
71
|
+
secateurs export
|
72
|
+
|
73
|
+
Options:
|
74
|
+
-h, [--host=HOST]
|
75
|
+
# Default: localhost
|
76
|
+
-p, [--port=N]
|
77
|
+
# Default: 9200
|
78
|
+
-f, [--format=FORMAT]
|
79
|
+
# Default: ruby
|
80
|
+
# Possible values: ruby, json, yaml
|
81
|
+
-o, [--output=OUTPUT]
|
82
|
+
|
83
|
+
[--split], [--no-split]
|
84
|
+
[--verbose], [--no-verbose]
|
85
|
+
|
86
|
+
export index template to local.
|
87
|
+
```
|
88
|
+
|
89
|
+
```sh
|
90
|
+
$ secateurs export > Templatefile
|
91
|
+
|
92
|
+
$ secateurs export -o Templatefile
|
93
|
+
|
94
|
+
$ secateurs export --format json > Templatefile.json
|
95
|
+
|
96
|
+
$ secateurs export --split -o templates/Templatefile
|
97
|
+
|
98
|
+
$ ls templates/*
|
99
|
+
Templatefile
|
100
|
+
template_1.rb
|
101
|
+
template_2.rb
|
102
|
+
|
103
|
+
$ cat templates/Templatefile
|
104
|
+
include_template "template_1"
|
105
|
+
include_template "template_2"
|
106
|
+
```
|
107
|
+
|
108
|
+
## DSL Statements
|
109
|
+
|
110
|
+
See [rails/jbuilder](https://github.com/rails/jbuilder).
|
111
|
+
|
112
|
+
## Index Template Examples
|
113
|
+
|
114
|
+
### Ruby DSL
|
115
|
+
|
116
|
+
```ruby
|
117
|
+
define_template "template_1" do
|
118
|
+
template "te*"
|
119
|
+
|
120
|
+
settings do
|
121
|
+
index do
|
122
|
+
number_of_shards "1"
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
mappings do
|
127
|
+
type1 do
|
128
|
+
_source do
|
129
|
+
enabled false
|
130
|
+
end
|
131
|
+
|
132
|
+
properties do
|
133
|
+
set! "@timestamp" do
|
134
|
+
type "date"
|
135
|
+
end
|
136
|
+
|
137
|
+
created_at do
|
138
|
+
format "EEE MMM dd HH:mm:ss Z YYYY"
|
139
|
+
type "date"
|
140
|
+
end
|
141
|
+
|
142
|
+
host_name do
|
143
|
+
index "not_analyzed"
|
144
|
+
type "string"
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
```
|
151
|
+
|
152
|
+
### JSON
|
153
|
+
|
154
|
+
```json
|
155
|
+
{
|
156
|
+
"template_1": {
|
157
|
+
"template": "te*",
|
158
|
+
"settings": {
|
159
|
+
"index": {
|
160
|
+
"number_of_shards": "1"
|
161
|
+
}
|
162
|
+
},
|
163
|
+
"mappings": {
|
164
|
+
"type1": {
|
165
|
+
"_source": {
|
166
|
+
"enabled": false
|
167
|
+
},
|
168
|
+
"properties": {
|
169
|
+
"@timestamp": {
|
170
|
+
"type": "date"
|
171
|
+
},
|
172
|
+
"created_at": {
|
173
|
+
"format": "EEE MMM dd HH:mm:ss Z YYYY",
|
174
|
+
"type": "date"
|
175
|
+
},
|
176
|
+
"host_name": {
|
177
|
+
"index": "not_analyzed",
|
178
|
+
"type": "string"
|
179
|
+
}
|
180
|
+
}
|
181
|
+
}
|
182
|
+
}
|
183
|
+
}
|
184
|
+
}
|
185
|
+
```
|
186
|
+
|
187
|
+
### YAML
|
188
|
+
|
189
|
+
```yaml
|
190
|
+
---
|
191
|
+
template_1:
|
192
|
+
template: te*
|
193
|
+
settings:
|
194
|
+
index:
|
195
|
+
number_of_shards: '1'
|
196
|
+
mappings:
|
197
|
+
type1:
|
198
|
+
_source:
|
199
|
+
enabled: false
|
200
|
+
properties:
|
201
|
+
"@timestamp":
|
202
|
+
type: date
|
203
|
+
created_at:
|
204
|
+
format: EEE MMM dd HH:mm:ss Z YYYY
|
205
|
+
type: date
|
206
|
+
host_name:
|
207
|
+
index: not_analyzed
|
208
|
+
type: string
|
209
|
+
```
|
210
|
+
|
211
|
+
### Partial Template
|
212
|
+
|
213
|
+
```ruby
|
214
|
+
def test_props(key)
|
215
|
+
set! key do
|
216
|
+
test true
|
217
|
+
end
|
218
|
+
end
|
219
|
+
# or
|
220
|
+
test_props2 = -> {
|
221
|
+
test2 true
|
222
|
+
}
|
223
|
+
# or
|
224
|
+
test_props3 = ->(json) {
|
225
|
+
json.test3 true
|
226
|
+
}
|
227
|
+
|
228
|
+
define_template "template_1" do
|
229
|
+
template "te*"
|
230
|
+
|
231
|
+
mappings do
|
232
|
+
partial! method(:test_props), "test-key"
|
233
|
+
|
234
|
+
type1 do
|
235
|
+
partial! test_props2
|
236
|
+
end
|
237
|
+
|
238
|
+
type2 do
|
239
|
+
test_props3.(self)
|
240
|
+
end
|
241
|
+
end
|
242
|
+
end
|
243
|
+
```
|
244
|
+
|
245
|
+
### Including Template
|
246
|
+
|
247
|
+
```ruby
|
248
|
+
include_template "/path/to/template_file"
|
249
|
+
|
250
|
+
# You can include a template by relative path.
|
251
|
+
include_template "templae_1"
|
252
|
+
```
|
253
|
+
|
254
|
+
## Development
|
255
|
+
|
256
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
257
|
+
|
258
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
259
|
+
|
260
|
+
## Contributing
|
261
|
+
|
262
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/daichirata/secateurs.
|
263
|
+
|
data/Rakefile
ADDED
data/bin/secateurs
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
jmx_props = ->(field) do
|
2
|
+
set! field do
|
3
|
+
properties do
|
4
|
+
OneMinuteRate do
|
5
|
+
type "float"
|
6
|
+
doc_values "true"
|
7
|
+
end
|
8
|
+
|
9
|
+
Count do
|
10
|
+
type "float"
|
11
|
+
doc_values "true"
|
12
|
+
end
|
13
|
+
|
14
|
+
FifteenMinuteRate do
|
15
|
+
type "float"
|
16
|
+
doc_values "true"
|
17
|
+
end
|
18
|
+
|
19
|
+
FiveMinuteRate do
|
20
|
+
type "float"
|
21
|
+
doc_values "true"
|
22
|
+
end
|
23
|
+
|
24
|
+
MeanRate do
|
25
|
+
type "float"
|
26
|
+
doc_values "true"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
define_template "kafkabeat" do
|
33
|
+
template "kafkabeat-*"
|
34
|
+
|
35
|
+
settings do
|
36
|
+
index do
|
37
|
+
refresh_interval "5s"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
mappings do
|
42
|
+
_default_ do
|
43
|
+
dynamic_templates do
|
44
|
+
child! do
|
45
|
+
template1 do
|
46
|
+
match "*"
|
47
|
+
|
48
|
+
mapping do
|
49
|
+
ignore_above 1024
|
50
|
+
index "not_analyzed"
|
51
|
+
type "{dynamic_type}"
|
52
|
+
doc_values true
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
_all do
|
59
|
+
norms do
|
60
|
+
enabled false
|
61
|
+
end
|
62
|
+
|
63
|
+
enabled true
|
64
|
+
end
|
65
|
+
|
66
|
+
properties do
|
67
|
+
set! "@timestamp" do
|
68
|
+
type "date"
|
69
|
+
end
|
70
|
+
|
71
|
+
jmx do
|
72
|
+
properties do
|
73
|
+
partial! jmx_props, "MessagesInPerSec"
|
74
|
+
partial! jmx_props, "BytesOutPerSec"
|
75
|
+
partial! jmx_props, "BytesInPerSec"
|
76
|
+
partial! jmx_props, "FailedProduceRequestsPerSec"
|
77
|
+
partial! jmx_props, "FailedFetchRequestsPerSec"
|
78
|
+
partial! jmx_props, "BytesRejectedPerSec"
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
define_template "template_1" do
|
2
|
+
template "te*"
|
3
|
+
|
4
|
+
settings do
|
5
|
+
index do
|
6
|
+
number_of_shards "1"
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
mappings do
|
11
|
+
type1 do
|
12
|
+
_source do
|
13
|
+
enabled false
|
14
|
+
end
|
15
|
+
|
16
|
+
properties do
|
17
|
+
set! "@timestamp" do
|
18
|
+
type "date"
|
19
|
+
end
|
20
|
+
|
21
|
+
created_at do
|
22
|
+
format "EEE MMM dd HH:mm:ss Z YYYY"
|
23
|
+
type "date"
|
24
|
+
end
|
25
|
+
|
26
|
+
host_name do
|
27
|
+
index "not_analyzed"
|
28
|
+
type "string"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
define_template "template_bar" do
|
2
|
+
template "te*"
|
3
|
+
|
4
|
+
settings do
|
5
|
+
index do
|
6
|
+
number_of_shards "1"
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
aliases do
|
11
|
+
alias2 do
|
12
|
+
filter do
|
13
|
+
term do
|
14
|
+
user "kimchy"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
index_routing "kimchy"
|
18
|
+
search_routing "kimchy"
|
19
|
+
end
|
20
|
+
|
21
|
+
set! "{index}-alias" do
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
define_template "template_foo" do
|
2
|
+
template "te*"
|
3
|
+
|
4
|
+
settings do
|
5
|
+
index do
|
6
|
+
number_of_shards "1"
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
mappings do
|
11
|
+
type1 do
|
12
|
+
_source do
|
13
|
+
enabled false
|
14
|
+
end
|
15
|
+
|
16
|
+
properties do
|
17
|
+
set! "@timestamp" do
|
18
|
+
type "date"
|
19
|
+
end
|
20
|
+
|
21
|
+
created_at do
|
22
|
+
format "EEE MMM dd HH:mm:ss Z YYYY"
|
23
|
+
type "date"
|
24
|
+
end
|
25
|
+
|
26
|
+
host_name do
|
27
|
+
index "not_analyzed"
|
28
|
+
type "string"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/lib/secateurs.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
require "active_support/core_ext/string"
|
2
|
+
require "json"
|
3
|
+
require "yaml"
|
4
|
+
|
5
|
+
require "secateurs/client"
|
6
|
+
require "secateurs/comparer"
|
7
|
+
require "secateurs/dsl"
|
8
|
+
require "secateurs/exporter"
|
9
|
+
require "secateurs/parser"
|
10
|
+
require "secateurs/runner"
|
11
|
+
require "secateurs/version"
|
12
|
+
|
13
|
+
module Secateurs
|
14
|
+
# Your code goes here...
|
15
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require "thor"
|
2
|
+
require "secateurs"
|
3
|
+
|
4
|
+
module Secateurs
|
5
|
+
class CLI < Thor
|
6
|
+
class_option :verbose, type: :boolean
|
7
|
+
|
8
|
+
def self.define_default_options
|
9
|
+
option :host, type: :string, aliases: ["-h"], default: "localhost"
|
10
|
+
option :port, type: :numeric, aliases: ["-p"], default: 9200
|
11
|
+
option :format, type: :string, aliases: ["-f"], default: "ruby", enum: ["ruby", "json", "yaml"]
|
12
|
+
end
|
13
|
+
|
14
|
+
desc "apply FILE", "apply index template file."
|
15
|
+
define_default_options
|
16
|
+
option :color, type: :boolean, default: true
|
17
|
+
option :dry_run, type: :boolean, default: false
|
18
|
+
def apply(file)
|
19
|
+
raise "Please specify template file." if file.nil?
|
20
|
+
|
21
|
+
msg = "Apply `#{file}` to index template"
|
22
|
+
msg += " (dry-run)" if options[:dry_run]
|
23
|
+
puts msg
|
24
|
+
|
25
|
+
Runner.new(options).run(file)
|
26
|
+
end
|
27
|
+
|
28
|
+
desc "export", "export index template to local."
|
29
|
+
define_default_options
|
30
|
+
option :output, type: :string, aliases: ["-o"]
|
31
|
+
option :split, type: :boolean, default: false
|
32
|
+
def export
|
33
|
+
Exporter.build(options).export
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require "elasticsearch"
|
2
|
+
|
3
|
+
module Secateurs
|
4
|
+
class Client
|
5
|
+
def self.from_options(options)
|
6
|
+
self.new(options[:host], options[:port], options[:verbose])
|
7
|
+
end
|
8
|
+
|
9
|
+
def initialize(host, port, verbose = true)
|
10
|
+
@client = Elasticsearch::Client.new(host: host, port: port, log: verbose)
|
11
|
+
end
|
12
|
+
|
13
|
+
def get_template(name = nil)
|
14
|
+
@client.indices.get_template(name: name, ignore: [404]) || {}
|
15
|
+
end
|
16
|
+
|
17
|
+
def put_template(name, body)
|
18
|
+
@client.indices.put_template(name: name, body: body)
|
19
|
+
end
|
20
|
+
|
21
|
+
def delete_template(name)
|
22
|
+
@client.indices.delete_template(name: name)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require "diffy"
|
2
|
+
|
3
|
+
module Secateurs
|
4
|
+
class Comparer
|
5
|
+
def initialize(colorize = true)
|
6
|
+
@output_format = colorize ? :color : :text
|
7
|
+
end
|
8
|
+
|
9
|
+
def compare(old, new)
|
10
|
+
Diffy::Diff.new(
|
11
|
+
convert_json(old),
|
12
|
+
convert_json(new),
|
13
|
+
diff: "-u"
|
14
|
+
).to_s(@output_format)
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def convert_json(hash)
|
20
|
+
(hash ? JSON.pretty_generate(sort_by_key(hash)) : "") + "\n"
|
21
|
+
end
|
22
|
+
|
23
|
+
def sort_by_key(hash)
|
24
|
+
hash.keys.sort.reduce({}) do |seed, key|
|
25
|
+
seed[key] = case (value = hash[key])
|
26
|
+
when Hash
|
27
|
+
sort_by_key(value)
|
28
|
+
else
|
29
|
+
value
|
30
|
+
end
|
31
|
+
seed
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require "secateurs/dsl/converter"
|
2
|
+
require "secateurs/dsl/template_builder"
|
3
|
+
|
4
|
+
module Secateurs
|
5
|
+
class DSL
|
6
|
+
DEFAULT = {
|
7
|
+
"order" => 0,
|
8
|
+
"settings" => {},
|
9
|
+
"mappings" => {},
|
10
|
+
"aliases" => {}
|
11
|
+
}
|
12
|
+
|
13
|
+
INDENT = 2
|
14
|
+
|
15
|
+
def self.parse(path)
|
16
|
+
self.new(path).parse
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize(path)
|
20
|
+
@path = File.expand_path(path)
|
21
|
+
@templates = {}
|
22
|
+
@indent = -INDENT
|
23
|
+
end
|
24
|
+
|
25
|
+
def parse
|
26
|
+
include_template(@path)
|
27
|
+
|
28
|
+
@templates
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def define_template(name, &block)
|
34
|
+
builder = TemplateBuilder.new do |b|
|
35
|
+
b.instance_eval(&block)
|
36
|
+
end
|
37
|
+
|
38
|
+
@templates[name] = DEFAULT.merge(builder.attributes!)
|
39
|
+
end
|
40
|
+
|
41
|
+
def include_template(path)
|
42
|
+
expanded_path = File.expand_path(path, File.dirname(@path))
|
43
|
+
expanded_path += '.rb' unless File.exist?(expanded_path)
|
44
|
+
source = File.read(expanded_path)
|
45
|
+
|
46
|
+
with_indent do |i|
|
47
|
+
puts "Include template from #{expanded_path}".indent(i)
|
48
|
+
instance_eval(source, expanded_path, 1)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def with_indent(&block)
|
53
|
+
@indent += INDENT
|
54
|
+
block.call(@indent)
|
55
|
+
ensure
|
56
|
+
@indent -= INDENT
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
module Secateurs
|
2
|
+
class DSL
|
3
|
+
class Converter
|
4
|
+
INDENT = 2
|
5
|
+
|
6
|
+
def self.convert(templates)
|
7
|
+
templates.map do |name, template|
|
8
|
+
self.new(name, template).convert
|
9
|
+
end.join("\n")
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize(name, attributes)
|
13
|
+
@name = name
|
14
|
+
@attributes = attributes
|
15
|
+
end
|
16
|
+
|
17
|
+
def convert
|
18
|
+
<<-EOF
|
19
|
+
define_template "#{@name}" do
|
20
|
+
#{convert_hash(@attributes)}
|
21
|
+
end
|
22
|
+
EOF
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def convert_hash(hash, indent = INDENT)
|
28
|
+
return if hash.nil?
|
29
|
+
|
30
|
+
hash.map do |k, v|
|
31
|
+
"#{convert_hash_key(k)}#{convert_hash_value(k, v)}".indent(indent)
|
32
|
+
end.join("\n")
|
33
|
+
end
|
34
|
+
|
35
|
+
def convert_hash_key(key)
|
36
|
+
if key =~ /[@{}\-\.]/
|
37
|
+
"set! \"#{key}\""
|
38
|
+
else
|
39
|
+
key
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def convert_hash_value(key, value)
|
44
|
+
case value
|
45
|
+
when Hash
|
46
|
+
return " do\n#{convert_hash(value)}\nend"
|
47
|
+
when Array
|
48
|
+
return " do\n#{convert_array(value)}\nend"
|
49
|
+
end
|
50
|
+
|
51
|
+
result = if key =~ /[@{}\-\.]/
|
52
|
+
","
|
53
|
+
else
|
54
|
+
""
|
55
|
+
end
|
56
|
+
|
57
|
+
result += if value.instance_of?(String)
|
58
|
+
" \"#{value}\""
|
59
|
+
else
|
60
|
+
" #{value}"
|
61
|
+
end
|
62
|
+
|
63
|
+
result
|
64
|
+
end
|
65
|
+
|
66
|
+
def convert_array(array)
|
67
|
+
return if array.empty?
|
68
|
+
|
69
|
+
array.map do |v|
|
70
|
+
"child! do\n#{convert_hash(v)}\nend".indent(INDENT)
|
71
|
+
end.join("\n")
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require "secateurs/exporter/base"
|
2
|
+
require "secateurs/exporter/json_exporter"
|
3
|
+
require "secateurs/exporter/ruby_exporter"
|
4
|
+
require "secateurs/exporter/split_exporter"
|
5
|
+
require "secateurs/exporter/yaml_exporter"
|
6
|
+
|
7
|
+
module Secateurs
|
8
|
+
module Exporter
|
9
|
+
def self.build(options)
|
10
|
+
if options[:split]
|
11
|
+
return SplitExporter.new(options)
|
12
|
+
end
|
13
|
+
|
14
|
+
case options[:format]
|
15
|
+
when "json"
|
16
|
+
JSONExporter.new(options)
|
17
|
+
when "ruby"
|
18
|
+
RubyExporter.new(options)
|
19
|
+
when "yaml"
|
20
|
+
YAMLExporter.new(options)
|
21
|
+
else
|
22
|
+
raise ArgumentError, "Invalid format. #{options[:format]}"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Secateurs
|
2
|
+
module Exporter
|
3
|
+
class Base
|
4
|
+
def initialize(options)
|
5
|
+
@options = options
|
6
|
+
@client = Client.from_options(options)
|
7
|
+
end
|
8
|
+
|
9
|
+
def export
|
10
|
+
body = generate_body
|
11
|
+
|
12
|
+
if output_file
|
13
|
+
write(output_file, body)
|
14
|
+
|
15
|
+
puts "Export index template to #{File.expand_path(output_file)}"
|
16
|
+
else
|
17
|
+
puts body
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def generate_body
|
24
|
+
raise NotImplementedError
|
25
|
+
end
|
26
|
+
|
27
|
+
def output_file
|
28
|
+
@options[:output]
|
29
|
+
end
|
30
|
+
|
31
|
+
def templates
|
32
|
+
@templates ||= @client.get_template
|
33
|
+
end
|
34
|
+
|
35
|
+
def write(output_file, body)
|
36
|
+
File.open(output_file, "wb") do |file|
|
37
|
+
file << body
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Secateurs
|
2
|
+
module Exporter
|
3
|
+
class RubyExporter < Base
|
4
|
+
MAGIC_COMMENT = <<-EOS.strip_heredoc
|
5
|
+
# -*- mode: ruby -*-
|
6
|
+
# vi: set ft=ruby :
|
7
|
+
EOS
|
8
|
+
|
9
|
+
def generate_body
|
10
|
+
with_magic_comment(DSL::Converter.convert(templates))
|
11
|
+
end
|
12
|
+
|
13
|
+
def with_magic_comment(body)
|
14
|
+
if output_file && File.extname(output_file) == ".rb"
|
15
|
+
body
|
16
|
+
else
|
17
|
+
MAGIC_COMMENT + "\n" + body
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require "fileutils"
|
2
|
+
|
3
|
+
module Secateurs
|
4
|
+
module Exporter
|
5
|
+
class SplitExporter < RubyExporter
|
6
|
+
def export
|
7
|
+
export_splitting_templates
|
8
|
+
super
|
9
|
+
end
|
10
|
+
|
11
|
+
def generate_body
|
12
|
+
body = templates.keys.map do |n|
|
13
|
+
"include_template \"#{n}\"\n"
|
14
|
+
end.join
|
15
|
+
|
16
|
+
with_magic_comment(body)
|
17
|
+
end
|
18
|
+
|
19
|
+
def output_file
|
20
|
+
super || "Templatefile"
|
21
|
+
end
|
22
|
+
|
23
|
+
def output_dir
|
24
|
+
File.dirname(output_file)
|
25
|
+
end
|
26
|
+
|
27
|
+
def export_splitting_templates
|
28
|
+
FileUtils.mkdir_p(output_dir)
|
29
|
+
|
30
|
+
templates.each do |name, template|
|
31
|
+
body = DSL::Converter.convert({ name => template })
|
32
|
+
output_file = File.join(output_dir, name + ".rb")
|
33
|
+
|
34
|
+
write(output_file, body)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Secateurs
|
2
|
+
class Parser
|
3
|
+
def initialize(format)
|
4
|
+
@parser = build_parser(format)
|
5
|
+
end
|
6
|
+
|
7
|
+
def parse(path)
|
8
|
+
@parser.call(path)
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def build_parser(format)
|
14
|
+
case format
|
15
|
+
when "json"
|
16
|
+
->(path) { JSON.parse(File.read(path)) }
|
17
|
+
when "ruby"
|
18
|
+
->(path) { DSL.parse(path) }
|
19
|
+
when "yaml"
|
20
|
+
->(path) { YAML.load_file(path) }
|
21
|
+
else
|
22
|
+
raise ArgumentError, "Invalid format `#{format}`."
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module Secateurs
|
2
|
+
class Runner
|
3
|
+
def initialize(options)
|
4
|
+
@options = options
|
5
|
+
@client = Client.from_options(options)
|
6
|
+
end
|
7
|
+
|
8
|
+
def run(file)
|
9
|
+
es_templates = @client.get_template
|
10
|
+
templates = Parser.new(@options[:format]).parse(file)
|
11
|
+
|
12
|
+
changed = {}
|
13
|
+
deleted = []
|
14
|
+
|
15
|
+
comparer = Comparer.new(@options[:color])
|
16
|
+
|
17
|
+
(es_templates.keys | templates.keys).each do |name|
|
18
|
+
result = comparer.compare(es_templates[name], templates[name])
|
19
|
+
|
20
|
+
unless result.blank?
|
21
|
+
puts
|
22
|
+
puts "Comparing changes `#{name}`"
|
23
|
+
puts result
|
24
|
+
|
25
|
+
if templates[name].nil?
|
26
|
+
deleted << name
|
27
|
+
else
|
28
|
+
changed[name] = templates[name]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
unless @options[:dry_run]
|
34
|
+
changed.each do |name, template|
|
35
|
+
@client.put_template(name, template)
|
36
|
+
end
|
37
|
+
|
38
|
+
deleted.each do |name|
|
39
|
+
@client.delete_template(name)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
data/secateurs.gemspec
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'secateurs/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "secateurs"
|
8
|
+
spec.version = Secateurs::VERSION
|
9
|
+
spec.authors = ["Daichi HIRATA"]
|
10
|
+
spec.email = ["bunny.hop.md@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = "Secateurs is a tool to manage Elasticsearch Index Template."
|
13
|
+
spec.description = "Secateurs is a tool to manage Elasticsearch Index Template."
|
14
|
+
spec.homepage = "https://github.com/daichirata/secateurs"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
17
|
+
spec.bindir = "bin"
|
18
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_runtime_dependency "activesupport", "~> 4.0"
|
22
|
+
spec.add_runtime_dependency "diffy"
|
23
|
+
spec.add_runtime_dependency "elasticsearch", "~> 1.0"
|
24
|
+
spec.add_runtime_dependency "jbuilder", "~> 2.0"
|
25
|
+
spec.add_runtime_dependency "thor"
|
26
|
+
|
27
|
+
spec.add_development_dependency "bundler", "~> 1.11"
|
28
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
29
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
30
|
+
spec.add_development_dependency "pry-byebug"
|
31
|
+
end
|
data/wercker.yml
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
box: ruby:2.3.0
|
2
|
+
# Build definition
|
3
|
+
build:
|
4
|
+
steps:
|
5
|
+
- script:
|
6
|
+
name: update bundler
|
7
|
+
code: gem update bundler
|
8
|
+
|
9
|
+
# A step that executes `bundle install` command
|
10
|
+
- bundle-install:
|
11
|
+
jobs: 4
|
12
|
+
|
13
|
+
# A custom script step, name value is used in the UI
|
14
|
+
# and the code value contains the command that get executed
|
15
|
+
- script:
|
16
|
+
name: echo ruby information
|
17
|
+
code: |
|
18
|
+
echo "ruby version $(ruby --version) running"
|
19
|
+
echo "from location $(which ruby)"
|
20
|
+
echo -p "gem list: $(gem list)"
|
21
|
+
|
22
|
+
# Add more steps here:
|
23
|
+
- script:
|
24
|
+
name: rspec
|
25
|
+
code: bundle exec rspec
|
metadata
ADDED
@@ -0,0 +1,200 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: secateurs
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Daichi HIRATA
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-05-04 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '4.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '4.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: diffy
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: elasticsearch
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: jbuilder
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '2.0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '2.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: thor
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: bundler
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '1.11'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '1.11'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rake
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '10.0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '10.0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: rspec
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '3.0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '3.0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: pry-byebug
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
description: Secateurs is a tool to manage Elasticsearch Index Template.
|
140
|
+
email:
|
141
|
+
- bunny.hop.md@gmail.com
|
142
|
+
executables:
|
143
|
+
- secateurs
|
144
|
+
extensions: []
|
145
|
+
extra_rdoc_files: []
|
146
|
+
files:
|
147
|
+
- ".gitignore"
|
148
|
+
- ".rspec"
|
149
|
+
- Gemfile
|
150
|
+
- LICENSE.txt
|
151
|
+
- README.md
|
152
|
+
- Rakefile
|
153
|
+
- bin/secateurs
|
154
|
+
- examples/Templatefile
|
155
|
+
- examples/kafkabeat.rb
|
156
|
+
- examples/template_1.rb
|
157
|
+
- examples/templates/template_bar.rb
|
158
|
+
- examples/templates/template_foo.rb
|
159
|
+
- lib/secateurs.rb
|
160
|
+
- lib/secateurs/cli.rb
|
161
|
+
- lib/secateurs/client.rb
|
162
|
+
- lib/secateurs/comparer.rb
|
163
|
+
- lib/secateurs/dsl.rb
|
164
|
+
- lib/secateurs/dsl/converter.rb
|
165
|
+
- lib/secateurs/dsl/template_builder.rb
|
166
|
+
- lib/secateurs/exporter.rb
|
167
|
+
- lib/secateurs/exporter/base.rb
|
168
|
+
- lib/secateurs/exporter/json_exporter.rb
|
169
|
+
- lib/secateurs/exporter/ruby_exporter.rb
|
170
|
+
- lib/secateurs/exporter/split_exporter.rb
|
171
|
+
- lib/secateurs/exporter/yaml_exporter.rb
|
172
|
+
- lib/secateurs/parser.rb
|
173
|
+
- lib/secateurs/runner.rb
|
174
|
+
- lib/secateurs/version.rb
|
175
|
+
- secateurs.gemspec
|
176
|
+
- wercker.yml
|
177
|
+
homepage: https://github.com/daichirata/secateurs
|
178
|
+
licenses: []
|
179
|
+
metadata: {}
|
180
|
+
post_install_message:
|
181
|
+
rdoc_options: []
|
182
|
+
require_paths:
|
183
|
+
- lib
|
184
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
185
|
+
requirements:
|
186
|
+
- - ">="
|
187
|
+
- !ruby/object:Gem::Version
|
188
|
+
version: '0'
|
189
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
190
|
+
requirements:
|
191
|
+
- - ">="
|
192
|
+
- !ruby/object:Gem::Version
|
193
|
+
version: '0'
|
194
|
+
requirements: []
|
195
|
+
rubyforge_project:
|
196
|
+
rubygems_version: 2.4.5.1
|
197
|
+
signing_key:
|
198
|
+
specification_version: 4
|
199
|
+
summary: Secateurs is a tool to manage Elasticsearch Index Template.
|
200
|
+
test_files: []
|