oxidized 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +2 -0
- data/Gemfile +3 -0
- data/README.md +133 -0
- data/Rakefile +46 -0
- data/TODO.md +20 -0
- data/bin/oxidized +13 -0
- data/extra/rest_client.rb +25 -0
- data/extra/syslog.rb +110 -0
- data/lib/oxidized.rb +6 -0
- data/lib/oxidized/cli.rb +42 -0
- data/lib/oxidized/config.rb +55 -0
- data/lib/oxidized/config/vars.rb +10 -0
- data/lib/oxidized/core.rb +41 -0
- data/lib/oxidized/input/cli.rb +48 -0
- data/lib/oxidized/input/input.rb +19 -0
- data/lib/oxidized/input/ssh.rb +111 -0
- data/lib/oxidized/input/telnet.rb +145 -0
- data/lib/oxidized/job.rb +14 -0
- data/lib/oxidized/jobs.rb +24 -0
- data/lib/oxidized/log.rb +21 -0
- data/lib/oxidized/manager.rb +57 -0
- data/lib/oxidized/model/acos.rb +69 -0
- data/lib/oxidized/model/aireos.rb +55 -0
- data/lib/oxidized/model/aos.rb +38 -0
- data/lib/oxidized/model/aos7.rb +58 -0
- data/lib/oxidized/model/aosw.rb +43 -0
- data/lib/oxidized/model/eos.rb +32 -0
- data/lib/oxidized/model/fortios.rb +44 -0
- data/lib/oxidized/model/ios.rb +63 -0
- data/lib/oxidized/model/iosxr.rb +47 -0
- data/lib/oxidized/model/ironware.rb +33 -0
- data/lib/oxidized/model/junos.rb +56 -0
- data/lib/oxidized/model/model.rb +152 -0
- data/lib/oxidized/model/powerconnect.rb +38 -0
- data/lib/oxidized/model/procurve.rb +45 -0
- data/lib/oxidized/model/timos.rb +45 -0
- data/lib/oxidized/node.rb +169 -0
- data/lib/oxidized/node/stats.rb +33 -0
- data/lib/oxidized/nodes.rb +143 -0
- data/lib/oxidized/output/file.rb +42 -0
- data/lib/oxidized/output/git.rb +78 -0
- data/lib/oxidized/output/output.rb +5 -0
- data/lib/oxidized/source/csv.rb +42 -0
- data/lib/oxidized/source/source.rb +11 -0
- data/lib/oxidized/source/sql.rb +61 -0
- data/lib/oxidized/string.rb +13 -0
- data/lib/oxidized/worker.rb +54 -0
- data/oxidized.gemspec +20 -0
- data/spec/nodes_spec.rb +46 -0
- metadata +136 -0
@@ -0,0 +1,33 @@
|
|
1
|
+
module Oxidized
|
2
|
+
class Node
|
3
|
+
class Stats
|
4
|
+
MAX_STAT = 10
|
5
|
+
|
6
|
+
# @param [Job] job job whose information add to stats
|
7
|
+
# @return [void]
|
8
|
+
def add job
|
9
|
+
stat = {
|
10
|
+
:start => job.start,
|
11
|
+
:end => job.end,
|
12
|
+
:time => job.time,
|
13
|
+
}
|
14
|
+
@stats[job.status] ||= []
|
15
|
+
@stats[job.status].shift if @stats[job.status].size > MAX_STAT
|
16
|
+
@stats[job.status].push stat
|
17
|
+
end
|
18
|
+
|
19
|
+
# @param [Symbol] status stats for specific status
|
20
|
+
# @return [Hash,Array] Hash of stats for every status or Array of stats for specific status
|
21
|
+
def get status=nil
|
22
|
+
status ? @stats[status] : @stats
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def initialize
|
28
|
+
@stats = {}
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,143 @@
|
|
1
|
+
module Oxidized
|
2
|
+
require 'oxidized/node'
|
3
|
+
require 'ipaddr'
|
4
|
+
class Oxidized::NotSupported < OxidizedError; end
|
5
|
+
class Oxidized::NodeNotFound < OxidizedError; end
|
6
|
+
class Nodes < Array
|
7
|
+
attr_accessor :source
|
8
|
+
alias :put :unshift
|
9
|
+
def load node_want=nil
|
10
|
+
with_lock do
|
11
|
+
new = []
|
12
|
+
node_want_ip = (IPAddr.new(node_want) rescue false) if node_want
|
13
|
+
@source = CFG.source.default
|
14
|
+
Oxidized.mgr.add_source @source
|
15
|
+
Oxidized.mgr.source[@source].new.load.each do |node|
|
16
|
+
|
17
|
+
# we want to load specific node(s), not all of them
|
18
|
+
if node_want
|
19
|
+
next unless node_want_ip == node[:ip] or node_want.match(node[:name])
|
20
|
+
end
|
21
|
+
|
22
|
+
begin
|
23
|
+
_node = Node.new node
|
24
|
+
new.push _node
|
25
|
+
rescue ModelNotFound => err
|
26
|
+
Log.error "node %s raised %s with message '%s'" % [node, err.class, err.message]
|
27
|
+
rescue Resolv::ResolvError => err
|
28
|
+
Log.error "node %s is not resolvable, raised %s with message '%s'" % [node, err.class, err.message]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
size == 0 ? replace(new) : update_nodes(new)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def list
|
36
|
+
with_lock do
|
37
|
+
map { |e| e.serialize }
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def show node
|
42
|
+
with_lock do
|
43
|
+
i = find_node_index node
|
44
|
+
self[i].serialize
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def fetch node, group
|
49
|
+
with_lock do
|
50
|
+
i = find_node_index node
|
51
|
+
output = self[i].output.new
|
52
|
+
raise Oxidized::NotSupported unless output.respond_to? :fetch
|
53
|
+
output.fetch node, group
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# @param node [String] name of the node moved into the head of array
|
58
|
+
def next node, opt={}
|
59
|
+
if waiting.find_node_index(node)
|
60
|
+
with_lock do
|
61
|
+
n = del node
|
62
|
+
n.user = opt['user']
|
63
|
+
n.msg = opt['msg']
|
64
|
+
n.from = opt['from']
|
65
|
+
# set last job to nil so that the node is picked for immediate update
|
66
|
+
n.last = nil
|
67
|
+
put n
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
alias :top :next
|
72
|
+
|
73
|
+
# @return [String] node from the head of the array
|
74
|
+
def get
|
75
|
+
with_lock do
|
76
|
+
(self << shift).last
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
# @param node node whose index number in Nodes to find
|
81
|
+
# @return [Fixnum] index number of node in Nodes
|
82
|
+
def find_node_index node
|
83
|
+
find_index node or raise Oxidized::NodeNotFound, "unable to find '#{node}'"
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
def initialize opts={}
|
89
|
+
super()
|
90
|
+
node = opts.delete :node
|
91
|
+
@mutex= Mutex.new # we compete for the nodes with webapi thread
|
92
|
+
if nodes = opts.delete(:nodes)
|
93
|
+
replace nodes
|
94
|
+
else
|
95
|
+
load node
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def with_lock &block
|
100
|
+
@mutex.synchronize(&block)
|
101
|
+
end
|
102
|
+
|
103
|
+
def find_index node
|
104
|
+
index { |e| e.name == node }
|
105
|
+
end
|
106
|
+
|
107
|
+
# @param node node which is removed from nodes list
|
108
|
+
# @return [Node] deleted node
|
109
|
+
def del node
|
110
|
+
delete_at find_node_index(node)
|
111
|
+
end
|
112
|
+
|
113
|
+
# @return [Nodes] list of nodes running now
|
114
|
+
def running
|
115
|
+
Nodes.new :nodes => select { |node| node.running? }
|
116
|
+
end
|
117
|
+
|
118
|
+
# @return [Nodes] list of nodes waiting (not running)
|
119
|
+
def waiting
|
120
|
+
Nodes.new :nodes => select { |node| not node.running? }
|
121
|
+
end
|
122
|
+
|
123
|
+
# walks list of new nodes, if old node contains same name, adds last and
|
124
|
+
# stats information from old to new.
|
125
|
+
#
|
126
|
+
# @todo can we trust name to be unique identifier, what about when groups are used?
|
127
|
+
# @param [Array] nodes Array of nodes used to replace+update old
|
128
|
+
def update_nodes nodes
|
129
|
+
old = self.dup
|
130
|
+
replace(nodes)
|
131
|
+
each do |node|
|
132
|
+
begin
|
133
|
+
if i = old.find_node_index(node.name)
|
134
|
+
node.stats = old[i].stats
|
135
|
+
node.last = old[i].last
|
136
|
+
end
|
137
|
+
rescue Oxidized::NodeNotFound
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
end
|
143
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Oxidized
|
2
|
+
class OxidizedFile < Output
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@cfg = CFG.output.file
|
7
|
+
end
|
8
|
+
|
9
|
+
def setup
|
10
|
+
if @cfg.empty?
|
11
|
+
CFGS.user.output.file.directory = File.join(Config::Root, 'configs')
|
12
|
+
CFGS.save :user
|
13
|
+
raise NoConfig, 'no output file config, edit ~/.config/oxidized/config'
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def store node, data, opt={}
|
18
|
+
file = @cfg.directory
|
19
|
+
if opt[:group]
|
20
|
+
file = File.join File.dirname(file), opt[:group]
|
21
|
+
end
|
22
|
+
FileUtils.mkdir_p file
|
23
|
+
file = File.join file, node
|
24
|
+
open(file, 'w') { |fh| fh.write data }
|
25
|
+
end
|
26
|
+
|
27
|
+
def fetch node, group
|
28
|
+
cfg_dir = @cfg.directory
|
29
|
+
if group # group is explicitly defined by user
|
30
|
+
IO.readlines File.join(cfg_dir, group, node)
|
31
|
+
else
|
32
|
+
if File.exists? File.join(cfg_dir, node) # node configuration file is stored on base directory
|
33
|
+
IO.readlines File.join(cfg_dir, node)
|
34
|
+
else
|
35
|
+
path = Dir.glob File.join(cfg_dir, '**', node) # fetch node in all groups
|
36
|
+
open(path[0], 'r').readlines
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
module Oxidized
|
2
|
+
class Git < Output
|
3
|
+
begin
|
4
|
+
require 'rugged'
|
5
|
+
rescue LoadError
|
6
|
+
raise OxidizedError, 'rugged not found: sudo gem install rugged'
|
7
|
+
end
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@cfg = CFG.output.git
|
11
|
+
end
|
12
|
+
|
13
|
+
def setup
|
14
|
+
if @cfg.empty?
|
15
|
+
CFGS.user.output.git.user = 'Oxidized'
|
16
|
+
CFGS.user.output.git.email = 'o@example.com'
|
17
|
+
CFGS.user.output.git.repo = File.join(Config::Root, 'oxidized.git')
|
18
|
+
CFGS.save :user
|
19
|
+
raise NoConfig, 'no output git config, edit ~/.config/oxidized/config'
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def store file, data, opt={}
|
24
|
+
msg = opt[:msg]
|
25
|
+
user = (opt[:user] or @cfg.user)
|
26
|
+
email = (opt[:email] or @cfg.email)
|
27
|
+
repo = @cfg.repo
|
28
|
+
if opt[:group]
|
29
|
+
repo = File.join File.dirname(repo), opt[:group] + '.git'
|
30
|
+
end
|
31
|
+
begin
|
32
|
+
repo = Rugged::Repository.new repo
|
33
|
+
update_repo repo, file, data, msg, user, email
|
34
|
+
rescue Rugged::OSError, Rugged::RepositoryError
|
35
|
+
Rugged::Repository.init_at repo, :bare
|
36
|
+
retry
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def fetch node, group
|
41
|
+
begin
|
42
|
+
repo = @cfg.repo
|
43
|
+
if group
|
44
|
+
repo = File.join File.dirname(repo), group + '.git'
|
45
|
+
end
|
46
|
+
repo = Rugged::Repository.new repo
|
47
|
+
index = repo.index
|
48
|
+
index.read_tree repo.head.target.tree unless repo.empty?
|
49
|
+
repo.read(index.get(node)[:oid]).data
|
50
|
+
rescue
|
51
|
+
'node not found'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def update_repo repo, file, data, msg, user, email
|
58
|
+
oid = repo.write data, :blob
|
59
|
+
index = repo.index
|
60
|
+
index.read_tree repo.head.target.tree unless repo.empty?
|
61
|
+
|
62
|
+
tree_old = index.write_tree repo
|
63
|
+
index.add :path=>file, :oid=>oid, :mode=>0100644
|
64
|
+
tree_new = index.write_tree repo
|
65
|
+
|
66
|
+
if tree_old != tree_new
|
67
|
+
repo.config['user.name'] = user
|
68
|
+
repo.config['user.email'] = email
|
69
|
+
Rugged::Commit.create(repo,
|
70
|
+
:tree => index.write_tree(repo),
|
71
|
+
:message => msg,
|
72
|
+
:parents => repo.empty? ? [] : [repo.head.target].compact,
|
73
|
+
:update_ref => 'HEAD',
|
74
|
+
)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Oxidized
|
2
|
+
class CSV < Source
|
3
|
+
def initialize
|
4
|
+
@cfg = CFG.source.csv
|
5
|
+
super
|
6
|
+
end
|
7
|
+
|
8
|
+
def setup
|
9
|
+
if @cfg.empty?
|
10
|
+
CFGS.user.source.csv.file = File.join(Config::Root, 'router.db')
|
11
|
+
CFGS.user.source.csv.delimiter = /:/
|
12
|
+
CFGS.user.source.csv.map.name = 0
|
13
|
+
CFGS.user.source.csv.map.model = 1
|
14
|
+
CFGS.save :user
|
15
|
+
raise NoConfig, 'no source csv config, edit ~/.config/oxidized/config'
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def load
|
20
|
+
nodes = []
|
21
|
+
open(@cfg.file).each_line do |line|
|
22
|
+
data = line.chomp.split @cfg.delimiter
|
23
|
+
next if data.empty?
|
24
|
+
# map node parameters
|
25
|
+
keys = {}
|
26
|
+
@cfg.map.each do |key, position|
|
27
|
+
keys[key.to_sym] = data[position]
|
28
|
+
end
|
29
|
+
keys[:model] = map_model keys[:model] if keys.key? :model
|
30
|
+
|
31
|
+
# map node specific vars, empty value is considered as nil
|
32
|
+
vars = {}
|
33
|
+
@cfg.vars_map.each { |key, position| vars[key.to_sym] = data[position].to_s.empty? ? nil : data[position] }
|
34
|
+
keys[:vars] = vars unless vars.empty?
|
35
|
+
|
36
|
+
nodes << keys
|
37
|
+
end
|
38
|
+
nodes
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
module Oxidized
|
2
|
+
class SQL < Source
|
3
|
+
begin
|
4
|
+
require 'sequel'
|
5
|
+
rescue LoadError
|
6
|
+
raise OxidizedError, 'sequel not found: sudo gem install sequel'
|
7
|
+
end
|
8
|
+
|
9
|
+
def setup
|
10
|
+
if @cfg.empty?
|
11
|
+
CFGS.user.source.sql.adapter = 'sqlite'
|
12
|
+
CFGS.user.source.sql.database = File.join(Config::Root, 'sqlite.db')
|
13
|
+
CFGS.user.source.sql.table = 'devices'
|
14
|
+
CFGS.user.source.sql.map.name = 'name'
|
15
|
+
CFGS.user.source.sql.map.model = 'rancid'
|
16
|
+
CFGS.save :user
|
17
|
+
raise NoConfig, 'no source sql config, edit ~/.config/oxidized/config'
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def load
|
22
|
+
nodes = []
|
23
|
+
db = connect
|
24
|
+
query = db[@cfg.table.to_sym]
|
25
|
+
query = query.with_sql(@cfg.query) if @cfg.query?
|
26
|
+
query.each do |node|
|
27
|
+
# map node parameters
|
28
|
+
keys = {}
|
29
|
+
@cfg.map.each { |key, sql_column| keys[key.to_sym] = node[sql_column.to_sym] }
|
30
|
+
keys[:model] = map_model keys[:model] if keys.key? :model
|
31
|
+
|
32
|
+
# map node specific vars
|
33
|
+
vars = {}
|
34
|
+
@cfg.vars_map.each { |key, sql_column| vars[key.to_sym] = node[sql_column.to_sym] }
|
35
|
+
keys[:vars] = vars unless vars.empty?
|
36
|
+
|
37
|
+
nodes << keys
|
38
|
+
end
|
39
|
+
db.disconnect
|
40
|
+
nodes
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def initialize
|
46
|
+
super
|
47
|
+
@cfg = CFG.source.sql
|
48
|
+
end
|
49
|
+
|
50
|
+
def connect
|
51
|
+
Sequel.connect(:adapter => @cfg.adapter,
|
52
|
+
:host => @cfg.host?,
|
53
|
+
:user => @cfg.user?,
|
54
|
+
:password => @cfg.password?,
|
55
|
+
:database => @cfg.database)
|
56
|
+
rescue Sequel::AdapterNotFound => error
|
57
|
+
raise OxidizedError, "SQL adapter gem not installed: " + error.message
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Oxidized
|
2
|
+
# Used in models, contains convenience methods
|
3
|
+
class String < String
|
4
|
+
# @return [Oxidized::String] copy of self with last line removed
|
5
|
+
def cut_tail
|
6
|
+
Oxy::String.new each_line.to_a[0..-2].join
|
7
|
+
end
|
8
|
+
# @return [Oxidized::String] copy of self with first line removed
|
9
|
+
def cut_head
|
10
|
+
Oxy::String.new each_line.to_a[1..-1].join
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Oxidized
|
2
|
+
require 'oxidized/job'
|
3
|
+
require 'oxidized/jobs'
|
4
|
+
class Worker
|
5
|
+
def initialize nodes
|
6
|
+
@nodes = nodes
|
7
|
+
@jobs = Jobs.new CFG.threads, CFG.interval, @nodes
|
8
|
+
Thread.abort_on_exception = true
|
9
|
+
end
|
10
|
+
def work
|
11
|
+
ended = []
|
12
|
+
@jobs.delete_if { |job| ended << job if not job.alive? }
|
13
|
+
ended.each { |job| process job }
|
14
|
+
while @jobs.size < @jobs.want
|
15
|
+
Log.debug "Jobs #{@jobs.size}, Want: #{@jobs.want}"
|
16
|
+
# ask for next node in queue non destructive way
|
17
|
+
nextnode = @nodes.first
|
18
|
+
unless nextnode.last.nil?
|
19
|
+
break if nextnode.last.end + CFG.interval > Time.now.utc
|
20
|
+
end
|
21
|
+
# shift nodes and get the next node
|
22
|
+
node = @nodes.get
|
23
|
+
node.running? ? next : node.running = true
|
24
|
+
@jobs.push Job.new node
|
25
|
+
end
|
26
|
+
end
|
27
|
+
def process job
|
28
|
+
node = job.node
|
29
|
+
node.last = job
|
30
|
+
node.stats.add job
|
31
|
+
@jobs.duration job.time
|
32
|
+
node.running = false
|
33
|
+
if job.status == :success
|
34
|
+
msg = "update #{node.name}"
|
35
|
+
msg += " from #{node.from}" if node.from
|
36
|
+
msg += " with message '#{node.msg}'" if node.msg
|
37
|
+
node.output.new.store node.name, job.config,
|
38
|
+
:msg => msg, :user => node.user, :group => node.group
|
39
|
+
node.reset
|
40
|
+
else
|
41
|
+
msg = "#{node.name} status #{job.status}"
|
42
|
+
if node.retry < CFG.retries
|
43
|
+
node.retry += 1
|
44
|
+
msg += ", retry attempt #{node.retry}"
|
45
|
+
@nodes.next node.name
|
46
|
+
else
|
47
|
+
msg += ", retries exhausted, giving up"
|
48
|
+
node.retry = 0
|
49
|
+
end
|
50
|
+
Log.warn msg
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|