hoss 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +1 -1
- data/README.md +2 -2
- data/VERSION +1 -1
- data/hoss.gemspec +4 -0
- data/lib/hoss/main.rb +51 -2
- data/lib/hoss/metadata.rb +1 -1
- data/lib/hoss/mjolnir.rb +56 -0
- data/lib/hoss/reindex.rb +127 -0
- metadata +63 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2fbcffb164bffc8e6678fa05d7d7a9f586ecfa71
|
4
|
+
data.tar.gz: 3ec2425370b1f16df23886a7997542e7ff6db82e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 533ef7c673ff7363f4e3d9c9526ec4e6a348d96f8dcc1301cfb71281e061f3160f0382cb1f03ec859bc4dac80df930f02224d8bdf80971c34606756bb38744bd
|
7
|
+
data.tar.gz: 8c7c9f8eb0dc93c1e7f4e51454ce3ca98830429bff0ca210a04031d58c215d88900bdd8eb5691d92e9f0298443b80dcfd03c7f369cc0f143962c0660738e4666
|
data/LICENSE
CHANGED
data/README.md
CHANGED
@@ -1,3 +1,3 @@
|
|
1
|
-
# Hoss
|
1
|
+
# Hoss 
|
2
2
|
|
3
|
-
|
3
|
+
Index your Chef Server with Elasticsearch.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0
|
1
|
+
0.1.0
|
data/hoss.gemspec
CHANGED
@@ -14,6 +14,10 @@ Gem::Specification.new do |s|
|
|
14
14
|
s.description = Hoss::SUMMARY + '.'
|
15
15
|
|
16
16
|
s.add_runtime_dependency 'thor', '~> 0'
|
17
|
+
s.add_runtime_dependency 'pmap', '~> 1'
|
18
|
+
s.add_runtime_dependency 'slog', '~> 1'
|
19
|
+
s.add_runtime_dependency 'ridley', '~> 4.1'
|
20
|
+
s.add_runtime_dependency 'elasticsearch', '~> 1'
|
17
21
|
|
18
22
|
s.files = `git ls-files`.split("\n")
|
19
23
|
s.test_files = `git ls-files -- test/*`.split("\n")
|
data/lib/hoss/main.rb
CHANGED
@@ -1,15 +1,64 @@
|
|
1
|
-
require '
|
1
|
+
require 'pmap'
|
2
|
+
require 'ridley'
|
3
|
+
require 'elasticsearch'
|
2
4
|
|
3
5
|
require_relative 'metadata'
|
6
|
+
require_relative 'mjolnir'
|
7
|
+
require_relative 'reindex'
|
4
8
|
|
5
9
|
|
10
|
+
Celluloid.logger = nil
|
11
|
+
|
6
12
|
module Hoss
|
7
|
-
class Main <
|
13
|
+
class Main < Mjolnir
|
8
14
|
|
9
15
|
desc 'version', 'Echo the application version'
|
10
16
|
def version
|
11
17
|
puts VERSION
|
12
18
|
end
|
13
19
|
|
20
|
+
|
21
|
+
desc 'art', 'View the application art'
|
22
|
+
def art
|
23
|
+
puts "\n%s\n" % ART
|
24
|
+
end
|
25
|
+
|
26
|
+
|
27
|
+
desc 'reindex', '(Re)index Chef server objects into Elasticsearch'
|
28
|
+
option :knife_path, \
|
29
|
+
type: :string,
|
30
|
+
aliases: %w[ -k ],
|
31
|
+
desc: 'Path to a knife for Chef server to index'
|
32
|
+
option :elasticsearch_host, \
|
33
|
+
type: :string,
|
34
|
+
aliases: %w[ -e ],
|
35
|
+
desc: 'Set Elasticsearch host to connect client',
|
36
|
+
default: 'localhost:9200'
|
37
|
+
option :index_name, \
|
38
|
+
type: :string,
|
39
|
+
aliases: %w[ -i ],
|
40
|
+
desc: 'Set Elasticsearch index name for object documents',
|
41
|
+
default: 'chef'
|
42
|
+
option :chef_resources, \
|
43
|
+
type: :string,
|
44
|
+
aliases: %w[ -r ],
|
45
|
+
desc: 'Comma-separated list of Chef resources to index',
|
46
|
+
default: 'user,cookbook,data_bag,environment,role,client,node'
|
47
|
+
include_common_options
|
48
|
+
def reindex
|
49
|
+
ridley = Ridley.from_chef_config options.knife
|
50
|
+
|
51
|
+
elasticsearch = Elasticsearch::Client.new \
|
52
|
+
host: options.elasticsearch_host,
|
53
|
+
reload_connections: true,
|
54
|
+
reload_on_failure: true,
|
55
|
+
trace: options.trace?,
|
56
|
+
log: options.trace? ? log : nil
|
57
|
+
|
58
|
+
Reindex.new options.merge \
|
59
|
+
ridley: ridley,
|
60
|
+
elasticsearch: elasticsearch,
|
61
|
+
logger: log
|
62
|
+
end
|
14
63
|
end
|
15
64
|
end
|
data/lib/hoss/metadata.rb
CHANGED
@@ -4,7 +4,7 @@ module Hoss
|
|
4
4
|
VERSION = File.read(File.join(File.dirname(__FILE__), '..', '..', 'VERSION')).strip
|
5
5
|
|
6
6
|
# We don't really do all that much, be humble
|
7
|
-
SUMMARY = '
|
7
|
+
SUMMARY = 'Index your Chef Server with Elasticsearch'
|
8
8
|
|
9
9
|
# Your benevolent dictator for life
|
10
10
|
AUTHOR = 'Sean Clemmer'
|
data/lib/hoss/mjolnir.rb
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'slog'
|
2
|
+
require 'thor'
|
3
|
+
|
4
|
+
|
5
|
+
# Thor's hammer! Like Thor with better logging
|
6
|
+
class Mjolnir < Thor
|
7
|
+
|
8
|
+
# Common options for Thor commands
|
9
|
+
COMMON_OPTIONS = {
|
10
|
+
log: {
|
11
|
+
type: :string,
|
12
|
+
aliases: %w[ -l ],
|
13
|
+
desc: 'Log to file instead of STDOUT',
|
14
|
+
default: ENV['HOSS_LOG'] || nil
|
15
|
+
},
|
16
|
+
debug: {
|
17
|
+
type: :boolean,
|
18
|
+
aliases: %w[ -v ],
|
19
|
+
desc: 'Enable DEBUG-level logging',
|
20
|
+
default: ENV['HOSS_DEBUG'] || false
|
21
|
+
},
|
22
|
+
trace: {
|
23
|
+
type: :boolean,
|
24
|
+
aliases: %w[ -z ],
|
25
|
+
desc: 'Enable TRACE-level logging',
|
26
|
+
default: ENV['HOSS_TRACE'] || false
|
27
|
+
}
|
28
|
+
}
|
29
|
+
|
30
|
+
# Decorate Thor commands with the options above
|
31
|
+
def self.include_common_options
|
32
|
+
COMMON_OPTIONS.each do |name, spec|
|
33
|
+
option name, spec
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
|
38
|
+
no_commands do
|
39
|
+
|
40
|
+
# Construct a Logger given the command-line options
|
41
|
+
def log
|
42
|
+
return @logger if defined? @logger
|
43
|
+
level = :info
|
44
|
+
level = :debug if options.debug?
|
45
|
+
level = :trace if options.trace?
|
46
|
+
device = options.log || $stderr
|
47
|
+
pretty = device.tty? rescue false
|
48
|
+
@logger = Slog.new \
|
49
|
+
out: device,
|
50
|
+
level: level,
|
51
|
+
colorize: pretty,
|
52
|
+
prettify: pretty
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
data/lib/hoss/reindex.rb
ADDED
@@ -0,0 +1,127 @@
|
|
1
|
+
module Hoss
|
2
|
+
class Reindex
|
3
|
+
attr_reader :resources, :index, :es, :ridley, :log
|
4
|
+
|
5
|
+
def initialize opts
|
6
|
+
@resources = opts[:chef_resources]
|
7
|
+
@index = opts[:index_name]
|
8
|
+
@es = opts[:elasticsearch]
|
9
|
+
@ridley = opts[:ridley]
|
10
|
+
@log = opts[:logger]
|
11
|
+
|
12
|
+
log.debug \
|
13
|
+
event: 'reindex',
|
14
|
+
opts: opts
|
15
|
+
|
16
|
+
reindex_started = Time.now
|
17
|
+
|
18
|
+
reindex
|
19
|
+
|
20
|
+
log.info \
|
21
|
+
event: 'reindexed',
|
22
|
+
elapsed: Time.now - reindex_started
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
|
30
|
+
def reindex
|
31
|
+
resources.split(',').each do |resource|
|
32
|
+
started = Time.now
|
33
|
+
log.debug event: 'load_resource', resource: resource
|
34
|
+
|
35
|
+
ops = load_resource resource
|
36
|
+
|
37
|
+
log.info \
|
38
|
+
event: 'loaded_resource',
|
39
|
+
resource: resource,
|
40
|
+
elapsed: Time.now - started
|
41
|
+
|
42
|
+
|
43
|
+
started = Time.now
|
44
|
+
log.debug event: 'index_resource', resource: resource
|
45
|
+
|
46
|
+
index_resource resource, ops
|
47
|
+
|
48
|
+
log.info \
|
49
|
+
event: 'indexed_resource',
|
50
|
+
resource: resource,
|
51
|
+
elapsed: Time.now - started
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
|
57
|
+
def load_resource resource
|
58
|
+
ridley.send(resource.to_sym).all.pmap(8) do |r|
|
59
|
+
r = if resource == 'cookbook'
|
60
|
+
{ 'name' => r[0], 'versions' => r[1] }
|
61
|
+
else
|
62
|
+
deep_hash r.reload
|
63
|
+
end
|
64
|
+
|
65
|
+
r['type'] = r['chef_type'] || resource
|
66
|
+
r['id'] = r['name']
|
67
|
+
|
68
|
+
raise if r['id'].nil?
|
69
|
+
|
70
|
+
log.trace event: 'reload_object', object: r
|
71
|
+
|
72
|
+
{
|
73
|
+
index: {
|
74
|
+
_index: index,
|
75
|
+
_type: r['type'],
|
76
|
+
_id: r['id'],
|
77
|
+
data: r
|
78
|
+
}
|
79
|
+
}
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
|
84
|
+
|
85
|
+
def index_resource resource, ops, bulk_size=500
|
86
|
+
while body = ops.shift(bulk_size)
|
87
|
+
break if body.empty?
|
88
|
+
|
89
|
+
begin
|
90
|
+
log.trace \
|
91
|
+
event: 'bulk_index',
|
92
|
+
response: es.bulk(body: body),
|
93
|
+
resource: resource,
|
94
|
+
size: body.size
|
95
|
+
|
96
|
+
log.debug \
|
97
|
+
event: 'index_resource',
|
98
|
+
resource: resource,
|
99
|
+
size: body.size
|
100
|
+
|
101
|
+
rescue
|
102
|
+
log.error \
|
103
|
+
event: 'bulk_index_failed',
|
104
|
+
resource: resource,
|
105
|
+
size: body.size,
|
106
|
+
body: body.inspect
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
|
112
|
+
|
113
|
+
def deep_hash obj
|
114
|
+
if obj.respond_to? :to_hash
|
115
|
+
obj = obj.to_hash
|
116
|
+
obj.keys.each do |k|
|
117
|
+
obj[k] = deep_hash obj[k]
|
118
|
+
end
|
119
|
+
obj
|
120
|
+
else
|
121
|
+
obj
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
|
126
|
+
end
|
127
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hoss
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sean Clemmer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-03-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -24,7 +24,63 @@ dependencies:
|
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
|
-
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: pmap
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: slog
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: ridley
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '4.1'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '4.1'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: elasticsearch
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1'
|
83
|
+
description: Index your Chef Server with Elasticsearch.
|
28
84
|
email: sclemmer@bluejeans.com
|
29
85
|
executables:
|
30
86
|
- hoss
|
@@ -42,6 +98,8 @@ files:
|
|
42
98
|
- lib/hoss.rb
|
43
99
|
- lib/hoss/main.rb
|
44
100
|
- lib/hoss/metadata.rb
|
101
|
+
- lib/hoss/mjolnir.rb
|
102
|
+
- lib/hoss/reindex.rb
|
45
103
|
homepage: https://github.com/sczizzo/hoss
|
46
104
|
licenses:
|
47
105
|
- ISC
|
@@ -62,8 +120,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
62
120
|
version: '0'
|
63
121
|
requirements: []
|
64
122
|
rubyforge_project:
|
65
|
-
rubygems_version: 2.
|
123
|
+
rubygems_version: 2.4.5
|
66
124
|
signing_key:
|
67
125
|
specification_version: 4
|
68
|
-
summary:
|
126
|
+
summary: Index your Chef Server with Elasticsearch
|
69
127
|
test_files: []
|