oxidized 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +2 -0
  3. data/Gemfile +3 -0
  4. data/README.md +133 -0
  5. data/Rakefile +46 -0
  6. data/TODO.md +20 -0
  7. data/bin/oxidized +13 -0
  8. data/extra/rest_client.rb +25 -0
  9. data/extra/syslog.rb +110 -0
  10. data/lib/oxidized.rb +6 -0
  11. data/lib/oxidized/cli.rb +42 -0
  12. data/lib/oxidized/config.rb +55 -0
  13. data/lib/oxidized/config/vars.rb +10 -0
  14. data/lib/oxidized/core.rb +41 -0
  15. data/lib/oxidized/input/cli.rb +48 -0
  16. data/lib/oxidized/input/input.rb +19 -0
  17. data/lib/oxidized/input/ssh.rb +111 -0
  18. data/lib/oxidized/input/telnet.rb +145 -0
  19. data/lib/oxidized/job.rb +14 -0
  20. data/lib/oxidized/jobs.rb +24 -0
  21. data/lib/oxidized/log.rb +21 -0
  22. data/lib/oxidized/manager.rb +57 -0
  23. data/lib/oxidized/model/acos.rb +69 -0
  24. data/lib/oxidized/model/aireos.rb +55 -0
  25. data/lib/oxidized/model/aos.rb +38 -0
  26. data/lib/oxidized/model/aos7.rb +58 -0
  27. data/lib/oxidized/model/aosw.rb +43 -0
  28. data/lib/oxidized/model/eos.rb +32 -0
  29. data/lib/oxidized/model/fortios.rb +44 -0
  30. data/lib/oxidized/model/ios.rb +63 -0
  31. data/lib/oxidized/model/iosxr.rb +47 -0
  32. data/lib/oxidized/model/ironware.rb +33 -0
  33. data/lib/oxidized/model/junos.rb +56 -0
  34. data/lib/oxidized/model/model.rb +152 -0
  35. data/lib/oxidized/model/powerconnect.rb +38 -0
  36. data/lib/oxidized/model/procurve.rb +45 -0
  37. data/lib/oxidized/model/timos.rb +45 -0
  38. data/lib/oxidized/node.rb +169 -0
  39. data/lib/oxidized/node/stats.rb +33 -0
  40. data/lib/oxidized/nodes.rb +143 -0
  41. data/lib/oxidized/output/file.rb +42 -0
  42. data/lib/oxidized/output/git.rb +78 -0
  43. data/lib/oxidized/output/output.rb +5 -0
  44. data/lib/oxidized/source/csv.rb +42 -0
  45. data/lib/oxidized/source/source.rb +11 -0
  46. data/lib/oxidized/source/sql.rb +61 -0
  47. data/lib/oxidized/string.rb +13 -0
  48. data/lib/oxidized/worker.rb +54 -0
  49. data/oxidized.gemspec +20 -0
  50. data/spec/nodes_spec.rb +46 -0
  51. metadata +136 -0
@@ -0,0 +1,33 @@
1
+ module Oxidized
2
+ class Node
3
+ class Stats
4
+ MAX_STAT = 10
5
+
6
+ # @param [Job] job job whose information add to stats
7
+ # @return [void]
8
+ def add job
9
+ stat = {
10
+ :start => job.start,
11
+ :end => job.end,
12
+ :time => job.time,
13
+ }
14
+ @stats[job.status] ||= []
15
+ @stats[job.status].shift if @stats[job.status].size > MAX_STAT
16
+ @stats[job.status].push stat
17
+ end
18
+
19
+ # @param [Symbol] status stats for specific status
20
+ # @return [Hash,Array] Hash of stats for every status or Array of stats for specific status
21
+ def get status=nil
22
+ status ? @stats[status] : @stats
23
+ end
24
+
25
+ private
26
+
27
+ def initialize
28
+ @stats = {}
29
+ end
30
+
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,143 @@
1
+ module Oxidized
2
+ require 'oxidized/node'
3
+ require 'ipaddr'
4
+ class Oxidized::NotSupported < OxidizedError; end
5
+ class Oxidized::NodeNotFound < OxidizedError; end
6
+ class Nodes < Array
7
+ attr_accessor :source
8
+ alias :put :unshift
9
+ def load node_want=nil
10
+ with_lock do
11
+ new = []
12
+ node_want_ip = (IPAddr.new(node_want) rescue false) if node_want
13
+ @source = CFG.source.default
14
+ Oxidized.mgr.add_source @source
15
+ Oxidized.mgr.source[@source].new.load.each do |node|
16
+
17
+ # we want to load specific node(s), not all of them
18
+ if node_want
19
+ next unless node_want_ip == node[:ip] or node_want.match(node[:name])
20
+ end
21
+
22
+ begin
23
+ _node = Node.new node
24
+ new.push _node
25
+ rescue ModelNotFound => err
26
+ Log.error "node %s raised %s with message '%s'" % [node, err.class, err.message]
27
+ rescue Resolv::ResolvError => err
28
+ Log.error "node %s is not resolvable, raised %s with message '%s'" % [node, err.class, err.message]
29
+ end
30
+ end
31
+ size == 0 ? replace(new) : update_nodes(new)
32
+ end
33
+ end
34
+
35
+ def list
36
+ with_lock do
37
+ map { |e| e.serialize }
38
+ end
39
+ end
40
+
41
+ def show node
42
+ with_lock do
43
+ i = find_node_index node
44
+ self[i].serialize
45
+ end
46
+ end
47
+
48
+ def fetch node, group
49
+ with_lock do
50
+ i = find_node_index node
51
+ output = self[i].output.new
52
+ raise Oxidized::NotSupported unless output.respond_to? :fetch
53
+ output.fetch node, group
54
+ end
55
+ end
56
+
57
+ # @param node [String] name of the node moved into the head of array
58
+ def next node, opt={}
59
+ if waiting.find_node_index(node)
60
+ with_lock do
61
+ n = del node
62
+ n.user = opt['user']
63
+ n.msg = opt['msg']
64
+ n.from = opt['from']
65
+ # set last job to nil so that the node is picked for immediate update
66
+ n.last = nil
67
+ put n
68
+ end
69
+ end
70
+ end
71
+ alias :top :next
72
+
73
+ # @return [String] node from the head of the array
74
+ def get
75
+ with_lock do
76
+ (self << shift).last
77
+ end
78
+ end
79
+
80
+ # @param node node whose index number in Nodes to find
81
+ # @return [Fixnum] index number of node in Nodes
82
+ def find_node_index node
83
+ find_index node or raise Oxidized::NodeNotFound, "unable to find '#{node}'"
84
+ end
85
+
86
+ private
87
+
88
+ def initialize opts={}
89
+ super()
90
+ node = opts.delete :node
91
+ @mutex= Mutex.new # we compete for the nodes with webapi thread
92
+ if nodes = opts.delete(:nodes)
93
+ replace nodes
94
+ else
95
+ load node
96
+ end
97
+ end
98
+
99
+ def with_lock &block
100
+ @mutex.synchronize(&block)
101
+ end
102
+
103
+ def find_index node
104
+ index { |e| e.name == node }
105
+ end
106
+
107
+ # @param node node which is removed from nodes list
108
+ # @return [Node] deleted node
109
+ def del node
110
+ delete_at find_node_index(node)
111
+ end
112
+
113
+ # @return [Nodes] list of nodes running now
114
+ def running
115
+ Nodes.new :nodes => select { |node| node.running? }
116
+ end
117
+
118
+ # @return [Nodes] list of nodes waiting (not running)
119
+ def waiting
120
+ Nodes.new :nodes => select { |node| not node.running? }
121
+ end
122
+
123
+ # walks list of new nodes, if old node contains same name, adds last and
124
+ # stats information from old to new.
125
+ #
126
+ # @todo can we trust name to be unique identifier, what about when groups are used?
127
+ # @param [Array] nodes Array of nodes used to replace+update old
128
+ def update_nodes nodes
129
+ old = self.dup
130
+ replace(nodes)
131
+ each do |node|
132
+ begin
133
+ if i = old.find_node_index(node.name)
134
+ node.stats = old[i].stats
135
+ node.last = old[i].last
136
+ end
137
+ rescue Oxidized::NodeNotFound
138
+ end
139
+ end
140
+ end
141
+
142
+ end
143
+ end
@@ -0,0 +1,42 @@
1
+ module Oxidized
2
+ class OxidizedFile < Output
3
+ require 'fileutils'
4
+
5
+ def initialize
6
+ @cfg = CFG.output.file
7
+ end
8
+
9
+ def setup
10
+ if @cfg.empty?
11
+ CFGS.user.output.file.directory = File.join(Config::Root, 'configs')
12
+ CFGS.save :user
13
+ raise NoConfig, 'no output file config, edit ~/.config/oxidized/config'
14
+ end
15
+ end
16
+
17
+ def store node, data, opt={}
18
+ file = @cfg.directory
19
+ if opt[:group]
20
+ file = File.join File.dirname(file), opt[:group]
21
+ end
22
+ FileUtils.mkdir_p file
23
+ file = File.join file, node
24
+ open(file, 'w') { |fh| fh.write data }
25
+ end
26
+
27
+ def fetch node, group
28
+ cfg_dir = @cfg.directory
29
+ if group # group is explicitly defined by user
30
+ IO.readlines File.join(cfg_dir, group, node)
31
+ else
32
+ if File.exists? File.join(cfg_dir, node) # node configuration file is stored on base directory
33
+ IO.readlines File.join(cfg_dir, node)
34
+ else
35
+ path = Dir.glob File.join(cfg_dir, '**', node) # fetch node in all groups
36
+ open(path[0], 'r').readlines
37
+ end
38
+ end
39
+ end
40
+
41
+ end
42
+ end
@@ -0,0 +1,78 @@
1
+ module Oxidized
2
+ class Git < Output
3
+ begin
4
+ require 'rugged'
5
+ rescue LoadError
6
+ raise OxidizedError, 'rugged not found: sudo gem install rugged'
7
+ end
8
+
9
+ def initialize
10
+ @cfg = CFG.output.git
11
+ end
12
+
13
+ def setup
14
+ if @cfg.empty?
15
+ CFGS.user.output.git.user = 'Oxidized'
16
+ CFGS.user.output.git.email = 'o@example.com'
17
+ CFGS.user.output.git.repo = File.join(Config::Root, 'oxidized.git')
18
+ CFGS.save :user
19
+ raise NoConfig, 'no output git config, edit ~/.config/oxidized/config'
20
+ end
21
+ end
22
+
23
+ def store file, data, opt={}
24
+ msg = opt[:msg]
25
+ user = (opt[:user] or @cfg.user)
26
+ email = (opt[:email] or @cfg.email)
27
+ repo = @cfg.repo
28
+ if opt[:group]
29
+ repo = File.join File.dirname(repo), opt[:group] + '.git'
30
+ end
31
+ begin
32
+ repo = Rugged::Repository.new repo
33
+ update_repo repo, file, data, msg, user, email
34
+ rescue Rugged::OSError, Rugged::RepositoryError
35
+ Rugged::Repository.init_at repo, :bare
36
+ retry
37
+ end
38
+ end
39
+
40
+ def fetch node, group
41
+ begin
42
+ repo = @cfg.repo
43
+ if group
44
+ repo = File.join File.dirname(repo), group + '.git'
45
+ end
46
+ repo = Rugged::Repository.new repo
47
+ index = repo.index
48
+ index.read_tree repo.head.target.tree unless repo.empty?
49
+ repo.read(index.get(node)[:oid]).data
50
+ rescue
51
+ 'node not found'
52
+ end
53
+ end
54
+
55
+ private
56
+
57
+ def update_repo repo, file, data, msg, user, email
58
+ oid = repo.write data, :blob
59
+ index = repo.index
60
+ index.read_tree repo.head.target.tree unless repo.empty?
61
+
62
+ tree_old = index.write_tree repo
63
+ index.add :path=>file, :oid=>oid, :mode=>0100644
64
+ tree_new = index.write_tree repo
65
+
66
+ if tree_old != tree_new
67
+ repo.config['user.name'] = user
68
+ repo.config['user.email'] = email
69
+ Rugged::Commit.create(repo,
70
+ :tree => index.write_tree(repo),
71
+ :message => msg,
72
+ :parents => repo.empty? ? [] : [repo.head.target].compact,
73
+ :update_ref => 'HEAD',
74
+ )
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,5 @@
1
+ module Oxidized
2
+ class Output
3
+ class NoConfig < OxidizedError; end
4
+ end
5
+ end
@@ -0,0 +1,42 @@
1
+ module Oxidized
2
+ class CSV < Source
3
+ def initialize
4
+ @cfg = CFG.source.csv
5
+ super
6
+ end
7
+
8
+ def setup
9
+ if @cfg.empty?
10
+ CFGS.user.source.csv.file = File.join(Config::Root, 'router.db')
11
+ CFGS.user.source.csv.delimiter = /:/
12
+ CFGS.user.source.csv.map.name = 0
13
+ CFGS.user.source.csv.map.model = 1
14
+ CFGS.save :user
15
+ raise NoConfig, 'no source csv config, edit ~/.config/oxidized/config'
16
+ end
17
+ end
18
+
19
+ def load
20
+ nodes = []
21
+ open(@cfg.file).each_line do |line|
22
+ data = line.chomp.split @cfg.delimiter
23
+ next if data.empty?
24
+ # map node parameters
25
+ keys = {}
26
+ @cfg.map.each do |key, position|
27
+ keys[key.to_sym] = data[position]
28
+ end
29
+ keys[:model] = map_model keys[:model] if keys.key? :model
30
+
31
+ # map node specific vars, empty value is considered as nil
32
+ vars = {}
33
+ @cfg.vars_map.each { |key, position| vars[key.to_sym] = data[position].to_s.empty? ? nil : data[position] }
34
+ keys[:vars] = vars unless vars.empty?
35
+
36
+ nodes << keys
37
+ end
38
+ nodes
39
+ end
40
+
41
+ end
42
+ end
@@ -0,0 +1,11 @@
1
+ module Oxidized
2
+ class Source
3
+ class NoConfig < OxidizedError; end
4
+ def initialize
5
+ @map = (CFG.model_map or {})
6
+ end
7
+ def map_model model
8
+ @map.has_key?(model) ? @map[model] : model
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,61 @@
1
+ module Oxidized
2
+ class SQL < Source
3
+ begin
4
+ require 'sequel'
5
+ rescue LoadError
6
+ raise OxidizedError, 'sequel not found: sudo gem install sequel'
7
+ end
8
+
9
+ def setup
10
+ if @cfg.empty?
11
+ CFGS.user.source.sql.adapter = 'sqlite'
12
+ CFGS.user.source.sql.database = File.join(Config::Root, 'sqlite.db')
13
+ CFGS.user.source.sql.table = 'devices'
14
+ CFGS.user.source.sql.map.name = 'name'
15
+ CFGS.user.source.sql.map.model = 'rancid'
16
+ CFGS.save :user
17
+ raise NoConfig, 'no source sql config, edit ~/.config/oxidized/config'
18
+ end
19
+ end
20
+
21
+ def load
22
+ nodes = []
23
+ db = connect
24
+ query = db[@cfg.table.to_sym]
25
+ query = query.with_sql(@cfg.query) if @cfg.query?
26
+ query.each do |node|
27
+ # map node parameters
28
+ keys = {}
29
+ @cfg.map.each { |key, sql_column| keys[key.to_sym] = node[sql_column.to_sym] }
30
+ keys[:model] = map_model keys[:model] if keys.key? :model
31
+
32
+ # map node specific vars
33
+ vars = {}
34
+ @cfg.vars_map.each { |key, sql_column| vars[key.to_sym] = node[sql_column.to_sym] }
35
+ keys[:vars] = vars unless vars.empty?
36
+
37
+ nodes << keys
38
+ end
39
+ db.disconnect
40
+ nodes
41
+ end
42
+
43
+ private
44
+
45
+ def initialize
46
+ super
47
+ @cfg = CFG.source.sql
48
+ end
49
+
50
+ def connect
51
+ Sequel.connect(:adapter => @cfg.adapter,
52
+ :host => @cfg.host?,
53
+ :user => @cfg.user?,
54
+ :password => @cfg.password?,
55
+ :database => @cfg.database)
56
+ rescue Sequel::AdapterNotFound => error
57
+ raise OxidizedError, "SQL adapter gem not installed: " + error.message
58
+ end
59
+
60
+ end
61
+ end
@@ -0,0 +1,13 @@
1
+ module Oxidized
2
+ # Used in models, contains convenience methods
3
+ class String < String
4
+ # @return [Oxidized::String] copy of self with last line removed
5
+ def cut_tail
6
+ Oxy::String.new each_line.to_a[0..-2].join
7
+ end
8
+ # @return [Oxidized::String] copy of self with first line removed
9
+ def cut_head
10
+ Oxy::String.new each_line.to_a[1..-1].join
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,54 @@
1
+ module Oxidized
2
+ require 'oxidized/job'
3
+ require 'oxidized/jobs'
4
+ class Worker
5
+ def initialize nodes
6
+ @nodes = nodes
7
+ @jobs = Jobs.new CFG.threads, CFG.interval, @nodes
8
+ Thread.abort_on_exception = true
9
+ end
10
+ def work
11
+ ended = []
12
+ @jobs.delete_if { |job| ended << job if not job.alive? }
13
+ ended.each { |job| process job }
14
+ while @jobs.size < @jobs.want
15
+ Log.debug "Jobs #{@jobs.size}, Want: #{@jobs.want}"
16
+ # ask for next node in queue non destructive way
17
+ nextnode = @nodes.first
18
+ unless nextnode.last.nil?
19
+ break if nextnode.last.end + CFG.interval > Time.now.utc
20
+ end
21
+ # shift nodes and get the next node
22
+ node = @nodes.get
23
+ node.running? ? next : node.running = true
24
+ @jobs.push Job.new node
25
+ end
26
+ end
27
+ def process job
28
+ node = job.node
29
+ node.last = job
30
+ node.stats.add job
31
+ @jobs.duration job.time
32
+ node.running = false
33
+ if job.status == :success
34
+ msg = "update #{node.name}"
35
+ msg += " from #{node.from}" if node.from
36
+ msg += " with message '#{node.msg}'" if node.msg
37
+ node.output.new.store node.name, job.config,
38
+ :msg => msg, :user => node.user, :group => node.group
39
+ node.reset
40
+ else
41
+ msg = "#{node.name} status #{job.status}"
42
+ if node.retry < CFG.retries
43
+ node.retry += 1
44
+ msg += ", retry attempt #{node.retry}"
45
+ @nodes.next node.name
46
+ else
47
+ msg += ", retries exhausted, giving up"
48
+ node.retry = 0
49
+ end
50
+ Log.warn msg
51
+ end
52
+ end
53
+ end
54
+ end