hoss 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/VERSION +1 -1
- data/lib/hoss/index.rb +1 -0
- data/lib/hoss/index/chef.rb +143 -0
- data/lib/hoss/main.rb +6 -32
- data/lib/hoss/metadata.rb +1 -1
- data/lib/hoss/mjolnir.rb +16 -0
- metadata +6 -5
- data/lib/hoss/reindex.rb +0 -127
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 77e0879929d3f7b534cefa7604849ab0b2258545
|
|
4
|
+
data.tar.gz: f5e96bf56a174c4c30e1f4e30d169fd31671264e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 901bc1d0559a101fd0c96c8d87a41f5a80c2060a92b47010c4ff00420f51b343e7a9688ae27df4fc198f6394574eb516d4aec8dc410b545ea0cc1bb3c13ee0cf
|
|
7
|
+
data.tar.gz: 3d45142d95daa19df9ca0afc186f625c760c0adf8e1366676f1f6c17bd04e087b042b8ae91011e35f29fa24ddea92b951488fd2d4924a89101fc52fd4c131e75
|
data/README.md
CHANGED
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.
|
|
1
|
+
0.2.0
|
data/lib/hoss/index.rb
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
require_relative 'index/chef'
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
require 'pmap'
|
|
2
|
+
require 'ridley'
|
|
3
|
+
require 'elasticsearch'
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
Celluloid.logger = nil
|
|
7
|
+
|
|
8
|
+
module Hoss
|
|
9
|
+
module Index
|
|
10
|
+
class Chef
|
|
11
|
+
attr_reader :resources, :index, :es, :ridley, :log
|
|
12
|
+
|
|
13
|
+
def initialize opts
|
|
14
|
+
@resources = opts[:chef_resources]
|
|
15
|
+
@index = opts[:index_name]
|
|
16
|
+
@log = opts[:logger]
|
|
17
|
+
|
|
18
|
+
@ridley = Ridley.from_chef_config opts.knife
|
|
19
|
+
|
|
20
|
+
@es = Elasticsearch::Client.new \
|
|
21
|
+
host: opts.elasticsearch_host,
|
|
22
|
+
reload_connections: true,
|
|
23
|
+
reload_on_failure: true,
|
|
24
|
+
trace: opts.trace?,
|
|
25
|
+
log: opts.trace? ? @log : nil
|
|
26
|
+
|
|
27
|
+
log.debug \
|
|
28
|
+
event: 'reindex',
|
|
29
|
+
opts: opts
|
|
30
|
+
|
|
31
|
+
started = Time.now
|
|
32
|
+
|
|
33
|
+
reindex
|
|
34
|
+
|
|
35
|
+
log.info \
|
|
36
|
+
event: 'reindexed',
|
|
37
|
+
elapsed: Time.now - started
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def reindex
|
|
46
|
+
resources.split(',').each do |resource|
|
|
47
|
+
started = Time.now
|
|
48
|
+
log.debug event: 'load_resource', resource: resource
|
|
49
|
+
|
|
50
|
+
ops = load_resource resource
|
|
51
|
+
|
|
52
|
+
log.info \
|
|
53
|
+
event: 'loaded_resource',
|
|
54
|
+
resource: resource,
|
|
55
|
+
elapsed: Time.now - started
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
started = Time.now
|
|
59
|
+
log.debug event: 'index_resource', resource: resource
|
|
60
|
+
|
|
61
|
+
index_resource resource, ops
|
|
62
|
+
|
|
63
|
+
log.info \
|
|
64
|
+
event: 'indexed_resource',
|
|
65
|
+
resource: resource,
|
|
66
|
+
elapsed: Time.now - started
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def load_resource resource
|
|
73
|
+
ridley.send(resource.to_sym).all.pmap(8) do |r|
|
|
74
|
+
r = if resource == 'cookbook'
|
|
75
|
+
{ 'name' => r[0], 'versions' => r[1] }
|
|
76
|
+
else
|
|
77
|
+
deep_hash r.reload
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
r['type'] = r['chef_type'] || resource
|
|
81
|
+
r['id'] = r['name']
|
|
82
|
+
|
|
83
|
+
raise if r['id'].nil?
|
|
84
|
+
|
|
85
|
+
log.trace event: 'reload_object', object: r
|
|
86
|
+
|
|
87
|
+
{
|
|
88
|
+
index: {
|
|
89
|
+
_index: index,
|
|
90
|
+
_type: r['type'],
|
|
91
|
+
_id: r['id'],
|
|
92
|
+
data: r
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def index_resource resource, ops, bulk_size=500
|
|
101
|
+
while body = ops.shift(bulk_size)
|
|
102
|
+
break if body.empty?
|
|
103
|
+
|
|
104
|
+
begin
|
|
105
|
+
log.trace \
|
|
106
|
+
event: 'bulk_index',
|
|
107
|
+
response: es.bulk(body: body),
|
|
108
|
+
resource: resource,
|
|
109
|
+
size: body.size
|
|
110
|
+
|
|
111
|
+
log.debug \
|
|
112
|
+
event: 'index_resource',
|
|
113
|
+
resource: resource,
|
|
114
|
+
size: body.size
|
|
115
|
+
|
|
116
|
+
rescue
|
|
117
|
+
log.error \
|
|
118
|
+
event: 'bulk_index_failed',
|
|
119
|
+
resource: resource,
|
|
120
|
+
size: body.size,
|
|
121
|
+
body: body.inspect
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def deep_hash obj
|
|
129
|
+
if obj.respond_to? :to_hash
|
|
130
|
+
obj = obj.to_hash
|
|
131
|
+
obj.keys.each do |k|
|
|
132
|
+
obj[k] = deep_hash obj[k]
|
|
133
|
+
end
|
|
134
|
+
obj
|
|
135
|
+
else
|
|
136
|
+
obj
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
data/lib/hoss/main.rb
CHANGED
|
@@ -1,13 +1,7 @@
|
|
|
1
|
-
require 'pmap'
|
|
2
|
-
require 'ridley'
|
|
3
|
-
require 'elasticsearch'
|
|
4
|
-
|
|
5
1
|
require_relative 'metadata'
|
|
6
2
|
require_relative 'mjolnir'
|
|
7
|
-
require_relative '
|
|
8
|
-
|
|
3
|
+
require_relative 'index'
|
|
9
4
|
|
|
10
|
-
Celluloid.logger = nil
|
|
11
5
|
|
|
12
6
|
module Hoss
|
|
13
7
|
class Main < Mjolnir
|
|
@@ -24,41 +18,21 @@ module Hoss
|
|
|
24
18
|
end
|
|
25
19
|
|
|
26
20
|
|
|
27
|
-
desc '
|
|
21
|
+
desc 'chef', '(Re)index Chef server objects into Elasticsearch'
|
|
28
22
|
option :knife_path, \
|
|
29
23
|
type: :string,
|
|
30
24
|
aliases: %w[ -k ],
|
|
31
25
|
desc: 'Path to a knife for Chef server to index'
|
|
32
|
-
option :elasticsearch_host, \
|
|
33
|
-
type: :string,
|
|
34
|
-
aliases: %w[ -e ],
|
|
35
|
-
desc: 'Set Elasticsearch host to connect client',
|
|
36
|
-
default: 'localhost:9200'
|
|
37
|
-
option :index_name, \
|
|
38
|
-
type: :string,
|
|
39
|
-
aliases: %w[ -i ],
|
|
40
|
-
desc: 'Set Elasticsearch index name for object documents',
|
|
41
|
-
default: 'chef'
|
|
42
26
|
option :chef_resources, \
|
|
43
27
|
type: :string,
|
|
44
28
|
aliases: %w[ -r ],
|
|
45
29
|
desc: 'Comma-separated list of Chef resources to index',
|
|
46
30
|
default: 'user,cookbook,data_bag,environment,role,client,node'
|
|
31
|
+
include_index_options
|
|
47
32
|
include_common_options
|
|
48
|
-
def
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
elasticsearch = Elasticsearch::Client.new \
|
|
52
|
-
host: options.elasticsearch_host,
|
|
53
|
-
reload_connections: true,
|
|
54
|
-
reload_on_failure: true,
|
|
55
|
-
trace: options.trace?,
|
|
56
|
-
log: options.trace? ? log : nil
|
|
57
|
-
|
|
58
|
-
Reindex.new options.merge \
|
|
59
|
-
ridley: ridley,
|
|
60
|
-
elasticsearch: elasticsearch,
|
|
61
|
-
logger: log
|
|
33
|
+
def index_chef
|
|
34
|
+
Hoss::Index::Chef.new options.merge(logger: log)
|
|
62
35
|
end
|
|
36
|
+
|
|
63
37
|
end
|
|
64
38
|
end
|
data/lib/hoss/metadata.rb
CHANGED
|
@@ -4,7 +4,7 @@ module Hoss
|
|
|
4
4
|
VERSION = File.read(File.join(File.dirname(__FILE__), '..', '..', 'VERSION')).strip
|
|
5
5
|
|
|
6
6
|
# We don't really do all that much, be humble
|
|
7
|
-
SUMMARY = 'Index your
|
|
7
|
+
SUMMARY = 'Index your infrastructure with Elasticsearch'
|
|
8
8
|
|
|
9
9
|
# Your benevolent dictator for life
|
|
10
10
|
AUTHOR = 'Sean Clemmer'
|
data/lib/hoss/mjolnir.rb
CHANGED
|
@@ -35,6 +35,22 @@ class Mjolnir < Thor
|
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
|
|
38
|
+
|
|
39
|
+
def self.include_index_options
|
|
40
|
+
option :elasticsearch_host, \
|
|
41
|
+
type: :string,
|
|
42
|
+
aliases: %w[ -e ],
|
|
43
|
+
desc: 'Set Elasticsearch host to connect client',
|
|
44
|
+
default: 'localhost:9200'
|
|
45
|
+
option :index_name, \
|
|
46
|
+
type: :string,
|
|
47
|
+
aliases: %w[ -i ],
|
|
48
|
+
desc: 'Set Elasticsearch index name for object documents',
|
|
49
|
+
default: 'chef'
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
|
|
38
54
|
no_commands do
|
|
39
55
|
|
|
40
56
|
# Construct a Logger given the command-line options
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: hoss
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sean Clemmer
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2015-03-
|
|
11
|
+
date: 2015-03-23 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: thor
|
|
@@ -80,7 +80,7 @@ dependencies:
|
|
|
80
80
|
- - "~>"
|
|
81
81
|
- !ruby/object:Gem::Version
|
|
82
82
|
version: '1'
|
|
83
|
-
description: Index your
|
|
83
|
+
description: Index your infrastructure with Elasticsearch.
|
|
84
84
|
email: sclemmer@bluejeans.com
|
|
85
85
|
executables:
|
|
86
86
|
- hoss
|
|
@@ -96,10 +96,11 @@ files:
|
|
|
96
96
|
- bin/hoss
|
|
97
97
|
- hoss.gemspec
|
|
98
98
|
- lib/hoss.rb
|
|
99
|
+
- lib/hoss/index.rb
|
|
100
|
+
- lib/hoss/index/chef.rb
|
|
99
101
|
- lib/hoss/main.rb
|
|
100
102
|
- lib/hoss/metadata.rb
|
|
101
103
|
- lib/hoss/mjolnir.rb
|
|
102
|
-
- lib/hoss/reindex.rb
|
|
103
104
|
homepage: https://github.com/sczizzo/hoss
|
|
104
105
|
licenses:
|
|
105
106
|
- ISC
|
|
@@ -123,5 +124,5 @@ rubyforge_project:
|
|
|
123
124
|
rubygems_version: 2.4.5
|
|
124
125
|
signing_key:
|
|
125
126
|
specification_version: 4
|
|
126
|
-
summary: Index your
|
|
127
|
+
summary: Index your infrastructure with Elasticsearch
|
|
127
128
|
test_files: []
|
data/lib/hoss/reindex.rb
DELETED
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
module Hoss
|
|
2
|
-
class Reindex
|
|
3
|
-
attr_reader :resources, :index, :es, :ridley, :log
|
|
4
|
-
|
|
5
|
-
def initialize opts
|
|
6
|
-
@resources = opts[:chef_resources]
|
|
7
|
-
@index = opts[:index_name]
|
|
8
|
-
@es = opts[:elasticsearch]
|
|
9
|
-
@ridley = opts[:ridley]
|
|
10
|
-
@log = opts[:logger]
|
|
11
|
-
|
|
12
|
-
log.debug \
|
|
13
|
-
event: 'reindex',
|
|
14
|
-
opts: opts
|
|
15
|
-
|
|
16
|
-
reindex_started = Time.now
|
|
17
|
-
|
|
18
|
-
reindex
|
|
19
|
-
|
|
20
|
-
log.info \
|
|
21
|
-
event: 'reindexed',
|
|
22
|
-
elapsed: Time.now - reindex_started
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
private
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def reindex
|
|
31
|
-
resources.split(',').each do |resource|
|
|
32
|
-
started = Time.now
|
|
33
|
-
log.debug event: 'load_resource', resource: resource
|
|
34
|
-
|
|
35
|
-
ops = load_resource resource
|
|
36
|
-
|
|
37
|
-
log.info \
|
|
38
|
-
event: 'loaded_resource',
|
|
39
|
-
resource: resource,
|
|
40
|
-
elapsed: Time.now - started
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
started = Time.now
|
|
44
|
-
log.debug event: 'index_resource', resource: resource
|
|
45
|
-
|
|
46
|
-
index_resource resource, ops
|
|
47
|
-
|
|
48
|
-
log.info \
|
|
49
|
-
event: 'indexed_resource',
|
|
50
|
-
resource: resource,
|
|
51
|
-
elapsed: Time.now - started
|
|
52
|
-
end
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def load_resource resource
|
|
58
|
-
ridley.send(resource.to_sym).all.pmap(8) do |r|
|
|
59
|
-
r = if resource == 'cookbook'
|
|
60
|
-
{ 'name' => r[0], 'versions' => r[1] }
|
|
61
|
-
else
|
|
62
|
-
deep_hash r.reload
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
r['type'] = r['chef_type'] || resource
|
|
66
|
-
r['id'] = r['name']
|
|
67
|
-
|
|
68
|
-
raise if r['id'].nil?
|
|
69
|
-
|
|
70
|
-
log.trace event: 'reload_object', object: r
|
|
71
|
-
|
|
72
|
-
{
|
|
73
|
-
index: {
|
|
74
|
-
_index: index,
|
|
75
|
-
_type: r['type'],
|
|
76
|
-
_id: r['id'],
|
|
77
|
-
data: r
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
end
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
def index_resource resource, ops, bulk_size=500
|
|
86
|
-
while body = ops.shift(bulk_size)
|
|
87
|
-
break if body.empty?
|
|
88
|
-
|
|
89
|
-
begin
|
|
90
|
-
log.trace \
|
|
91
|
-
event: 'bulk_index',
|
|
92
|
-
response: es.bulk(body: body),
|
|
93
|
-
resource: resource,
|
|
94
|
-
size: body.size
|
|
95
|
-
|
|
96
|
-
log.debug \
|
|
97
|
-
event: 'index_resource',
|
|
98
|
-
resource: resource,
|
|
99
|
-
size: body.size
|
|
100
|
-
|
|
101
|
-
rescue
|
|
102
|
-
log.error \
|
|
103
|
-
event: 'bulk_index_failed',
|
|
104
|
-
resource: resource,
|
|
105
|
-
size: body.size,
|
|
106
|
-
body: body.inspect
|
|
107
|
-
end
|
|
108
|
-
end
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
def deep_hash obj
|
|
114
|
-
if obj.respond_to? :to_hash
|
|
115
|
-
obj = obj.to_hash
|
|
116
|
-
obj.keys.each do |k|
|
|
117
|
-
obj[k] = deep_hash obj[k]
|
|
118
|
-
end
|
|
119
|
-
obj
|
|
120
|
-
else
|
|
121
|
-
obj
|
|
122
|
-
end
|
|
123
|
-
end
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
end
|
|
127
|
-
end
|