oxidized 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +2 -0
- data/Gemfile +3 -0
- data/README.md +133 -0
- data/Rakefile +46 -0
- data/TODO.md +20 -0
- data/bin/oxidized +13 -0
- data/extra/rest_client.rb +25 -0
- data/extra/syslog.rb +110 -0
- data/lib/oxidized.rb +6 -0
- data/lib/oxidized/cli.rb +42 -0
- data/lib/oxidized/config.rb +55 -0
- data/lib/oxidized/config/vars.rb +10 -0
- data/lib/oxidized/core.rb +41 -0
- data/lib/oxidized/input/cli.rb +48 -0
- data/lib/oxidized/input/input.rb +19 -0
- data/lib/oxidized/input/ssh.rb +111 -0
- data/lib/oxidized/input/telnet.rb +145 -0
- data/lib/oxidized/job.rb +14 -0
- data/lib/oxidized/jobs.rb +24 -0
- data/lib/oxidized/log.rb +21 -0
- data/lib/oxidized/manager.rb +57 -0
- data/lib/oxidized/model/acos.rb +69 -0
- data/lib/oxidized/model/aireos.rb +55 -0
- data/lib/oxidized/model/aos.rb +38 -0
- data/lib/oxidized/model/aos7.rb +58 -0
- data/lib/oxidized/model/aosw.rb +43 -0
- data/lib/oxidized/model/eos.rb +32 -0
- data/lib/oxidized/model/fortios.rb +44 -0
- data/lib/oxidized/model/ios.rb +63 -0
- data/lib/oxidized/model/iosxr.rb +47 -0
- data/lib/oxidized/model/ironware.rb +33 -0
- data/lib/oxidized/model/junos.rb +56 -0
- data/lib/oxidized/model/model.rb +152 -0
- data/lib/oxidized/model/powerconnect.rb +38 -0
- data/lib/oxidized/model/procurve.rb +45 -0
- data/lib/oxidized/model/timos.rb +45 -0
- data/lib/oxidized/node.rb +169 -0
- data/lib/oxidized/node/stats.rb +33 -0
- data/lib/oxidized/nodes.rb +143 -0
- data/lib/oxidized/output/file.rb +42 -0
- data/lib/oxidized/output/git.rb +78 -0
- data/lib/oxidized/output/output.rb +5 -0
- data/lib/oxidized/source/csv.rb +42 -0
- data/lib/oxidized/source/source.rb +11 -0
- data/lib/oxidized/source/sql.rb +61 -0
- data/lib/oxidized/string.rb +13 -0
- data/lib/oxidized/worker.rb +54 -0
- data/oxidized.gemspec +20 -0
- data/spec/nodes_spec.rb +46 -0
- metadata +136 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: e120e1cba4d53d4ddc38c897d735bc48af36cd3d
|
4
|
+
data.tar.gz: e02f9953659fa973777ff3a8120c9d4a28426e0e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 5f441707347f65ae7e4919ad7ffe392b807484740d59faf88f071c89ec04504ce5320e4211724945bb81c9ee577f239001c32bbf5364b5a6c587e8a5f0393e96
|
7
|
+
data.tar.gz: 8097bad5db23fedf183fa7e2ca17d238e199feb8862d5def0d04d4a3075d901db92729871b9adabdb42a412f5ee4bf4c7804d62a2d6533900b42f2275f23a63d
|
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,133 @@
|
|
1
|
+
# Pitch
|
2
|
+
* automatically adds/removes threads to meet configured retrieval interval
|
3
|
+
* restful API to move node immediately to head-of-queue (GET/POST /node/next/[NODE])
|
4
|
+
* syslog udp+file example to catch config change event (ios/junos) and trigger config fetch
|
5
|
+
* will signal ios/junos user who made change, which output module can (git does) use (via POST)
|
6
|
+
* 'git blame' will show for each line who and when the change was made
|
7
|
+
* restful API to reload list of nodes (GET /reload)
|
8
|
+
* restful API to fetch configurations (/node/fetch/[NODE] or /node/fetch/group/[NODE])
|
9
|
+
* restful API to show list of nodes (GET /nodes)
|
10
|
+
|
11
|
+
# Install
|
12
|
+
* early days, but try:
|
13
|
+
1. apt-get install ruby ruby-dev libsqlite3-dev libssl-dev
|
14
|
+
2. gem install oxidized
|
15
|
+
3. gem install oxidized-script oxidized-web # if you don't install oxidized-web, make sure you remove "rest" from your config
|
16
|
+
4. oxidized
|
17
|
+
5. vi ~/.config/oxidized/config
|
18
|
+
6. (maybe point to your rancid/router.db or copy it there)
|
19
|
+
7. oxidized
|
20
|
+
|
21
|
+
# API
|
22
|
+
## Input
|
23
|
+
* gets config from nodes
|
24
|
+
* must implement 'connect', 'get', 'cmd'
|
25
|
+
* 'ssh' and 'telnet' implemented
|
26
|
+
|
27
|
+
## Output
|
28
|
+
* stores config
|
29
|
+
* must implement 'store' (may implement 'fetch')
|
30
|
+
* 'git' and 'file' (store as flat ascii) implemented
|
31
|
+
|
32
|
+
## Source
|
33
|
+
* gets list of nodes to poll
|
34
|
+
* must implement 'load'
|
35
|
+
* source can have 'name', 'model', 'group', 'username', 'password', 'input', 'output', 'prompt'
|
36
|
+
* name - name of the devices
|
37
|
+
* model - model to use ios/junos/xyz, model is loaded dynamically when needed (Also default in config file)
|
38
|
+
* input - method to acquire config, loaded dynamically as needed (Also default in config file)
|
39
|
+
* output - method to store config, loaded dynamically as needed (Also default in config file)
|
40
|
+
* prompt - prompt used for node (Also default in config file, can be specified in model too)
|
41
|
+
* 'sql' and 'csv' (supports any format with single entry per line, like router.db)
|
42
|
+
|
43
|
+
## Model
|
44
|
+
* lists commands to gather from given device model
|
45
|
+
* can use 'cmd', 'prompt', 'comment', 'cfg'
|
46
|
+
* cfg is executed in input/output/source context
|
47
|
+
* cmd is executed in instance of model
|
48
|
+
* 'junos', 'ios', 'ironware' and 'powerconnect' implemented
|
49
|
+
|
50
|
+
## Media
|
51
|
+
* TREX 2014 presentation - http://youtu.be/kBQ_CTUuqeU#t=3h
|
52
|
+
|
53
|
+
## Cookbook
|
54
|
+
|
55
|
+
### Configuration I use in one environment
|
56
|
+
```
|
57
|
+
---
|
58
|
+
username: LANA
|
59
|
+
password: LANAAAAAAA
|
60
|
+
output:
|
61
|
+
default: git
|
62
|
+
git:
|
63
|
+
user: Oxidized
|
64
|
+
email: o@example.com
|
65
|
+
repo: "/usr/local/lan/oxidized.git"
|
66
|
+
source:
|
67
|
+
default: sql
|
68
|
+
sql:
|
69
|
+
adapter: sqlite
|
70
|
+
file: "/usr/local/lan/corona.db"
|
71
|
+
table: device
|
72
|
+
map:
|
73
|
+
name: ptr
|
74
|
+
model: model
|
75
|
+
```
|
76
|
+
|
77
|
+
### Configuration you end up after first run
|
78
|
+
If you don't configure output and source, it'll further fill them with example
|
79
|
+
configs for your chosen output/source in subsequent runs
|
80
|
+
```
|
81
|
+
---
|
82
|
+
username: username
|
83
|
+
password: password
|
84
|
+
model: junos
|
85
|
+
interval: 3600
|
86
|
+
log: "/home/fisakytt/.config/oxidized/log"
|
87
|
+
debug: false
|
88
|
+
threads: 30
|
89
|
+
timeout: 30
|
90
|
+
prompt: !ruby/regexp /^([\w.@-]+[#>]\s?)$/
|
91
|
+
rest: 127.0.0.1:8888
|
92
|
+
vars: {}
|
93
|
+
input:
|
94
|
+
default: ssh, telnet
|
95
|
+
ssh:
|
96
|
+
secure: false
|
97
|
+
output:
|
98
|
+
default: git
|
99
|
+
source:
|
100
|
+
default: csv
|
101
|
+
model_map:
|
102
|
+
cisco: ios
|
103
|
+
juniper: junos
|
104
|
+
```
|
105
|
+
|
106
|
+
Output and Source could be:
|
107
|
+
```
|
108
|
+
output:
|
109
|
+
default: git
|
110
|
+
git:
|
111
|
+
user: Oxidized
|
112
|
+
email: o@example.com
|
113
|
+
repo: "/home/fisakytt/.config/oxidized/oxidized.git"
|
114
|
+
source:
|
115
|
+
default: csv
|
116
|
+
csv:
|
117
|
+
file: "/home/fisakytt/.config/oxidized/router.db"
|
118
|
+
delimiter: !ruby/regexp /:/
|
119
|
+
map:
|
120
|
+
name: 0
|
121
|
+
model: 1
|
122
|
+
```
|
123
|
+
which reads nodes from rancid compatible router.db maps their model names to
|
124
|
+
model names oxidized expects, stores config in git, will try ssh first then
|
125
|
+
telnet, wont crash on changed ssh keys.
|
126
|
+
|
127
|
+
Hopefully most of them are obvious, log is ignored if Syslog::Logger exists
|
128
|
+
(>=2.0) and syslog is used instead.
|
129
|
+
|
130
|
+
System wide configurations can be stored in /etc/oxidized/config, this might be
|
131
|
+
useful for storing for example source information, if many users are using
|
132
|
+
oxs/Oxidized::Script, which would allow user specific config only to include
|
133
|
+
username+password.
|
data/Rakefile
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
begin
|
2
|
+
require 'rake/testtask'
|
3
|
+
require 'bundler'
|
4
|
+
# Bundler.setup
|
5
|
+
rescue LoadError
|
6
|
+
warn 'bunler missing'
|
7
|
+
end
|
8
|
+
|
9
|
+
gemspec = eval(File.read(Dir['*.gemspec'].first))
|
10
|
+
file = [gemspec.name, gemspec.version].join('-') + '.gem'
|
11
|
+
|
12
|
+
desc 'Validate gemspec'
|
13
|
+
task :gemspec do
|
14
|
+
gemspec.validate
|
15
|
+
end
|
16
|
+
|
17
|
+
desc 'Run minitest'
|
18
|
+
task :test do
|
19
|
+
Rake::TestTask.new do |t|
|
20
|
+
t.libs.push "lib"
|
21
|
+
t.test_files = FileList['spec/*_spec.rb']
|
22
|
+
t.verbose = true
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
desc 'Build gem'
|
27
|
+
task :build do
|
28
|
+
system "gem build #{gemspec.name}.gemspec"
|
29
|
+
FileUtils.mkdir_p 'gems'
|
30
|
+
FileUtils.mv file, 'gems'
|
31
|
+
end
|
32
|
+
|
33
|
+
desc 'Install gem'
|
34
|
+
task :install => :build do
|
35
|
+
system "sudo -Es sh -c \'umask 022; gem install gems/#{file}\'"
|
36
|
+
end
|
37
|
+
|
38
|
+
desc 'Remove gems'
|
39
|
+
task :clean do
|
40
|
+
FileUtils.rm_rf 'gems'
|
41
|
+
end
|
42
|
+
|
43
|
+
desc 'Push to rubygems'
|
44
|
+
task :push do
|
45
|
+
system "gem push gems/#{file}"
|
46
|
+
end
|
data/TODO.md
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# thread number
|
2
|
+
* think about algo
|
3
|
+
* if job ended later than now-iteration have rand(node.size) == 0 to add thread
|
4
|
+
* if now is less than job_ended+iteration same chance to remove thread?
|
5
|
+
* should we try to avoid max threads from being hit? (like maybe non-success thread is pulling average?)
|
6
|
+
|
7
|
+
# config
|
8
|
+
* save keys as strings, load as symbols?
|
9
|
+
|
10
|
+
# other
|
11
|
+
should it offer cli mass config-pusher? (I think not, I have ideas for such
|
12
|
+
program and I'm not sure if synergies are high enough for shared code without
|
13
|
+
making both bit awkward)
|
14
|
+
|
15
|
+
use sidekiq? Any benefits?
|
16
|
+
|
17
|
+
|
18
|
+
# docs, testing
|
19
|
+
* yard docs
|
20
|
+
* rspec tests
|
data/bin/oxidized
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
|
4
|
+
# FIX ME, killing oxidized needs -9
|
5
|
+
trap("INT") { exit } # sinatra will otherwise steal this from us
|
6
|
+
|
7
|
+
begin
|
8
|
+
require 'oxidized/cli'
|
9
|
+
Oxidized::CLI.new.run
|
10
|
+
rescue => error
|
11
|
+
warn "#{error}"
|
12
|
+
raise if Oxidized::CFG.debug
|
13
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Oxidized
|
2
|
+
class RestClient
|
3
|
+
require 'net/http'
|
4
|
+
require 'json'
|
5
|
+
HOST = 'localhost'
|
6
|
+
PORT = 8888
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def next opt={}, host=HOST, port=PORT
|
10
|
+
web = new host, port
|
11
|
+
web.next opt
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def initialize host=HOST, port=PORT
|
16
|
+
@web = Net::HTTP.new host, port
|
17
|
+
end
|
18
|
+
|
19
|
+
def next opt
|
20
|
+
data = JSON.dump opt
|
21
|
+
@web.put '/node/next/' + opt[:name].to_s, data
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
data/extra/syslog.rb
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# IOS:
|
4
|
+
# logging discriminator CFG mnemonics includes CONFIG_I
|
5
|
+
# logging host SERVER discriminator CFG
|
6
|
+
|
7
|
+
# JunOS:
|
8
|
+
# set system syslog host SERVER interactive-commands notice
|
9
|
+
# set system syslog host SERVER match "^mgd\[[0-9]+\]: UI_COMMIT: .*"
|
10
|
+
|
11
|
+
# sudo setcap 'cap_net_bind_service=+ep' /usr/bin/ruby
|
12
|
+
|
13
|
+
# exit if fork ## TODO: proper daemonize
|
14
|
+
|
15
|
+
require 'socket'
|
16
|
+
require 'resolv'
|
17
|
+
require './rest_client'
|
18
|
+
|
19
|
+
module Oxidized
|
20
|
+
class SyslogMonitor
|
21
|
+
NAME_MAP = {
|
22
|
+
/(.*)\.ip\.tdc\.net/ => '\1',
|
23
|
+
/(.*)\.ip\.fi/ => '\1',
|
24
|
+
}
|
25
|
+
PORT = 514
|
26
|
+
FILE = 'messages'
|
27
|
+
MSG = {
|
28
|
+
:ios => '%SYS-5-CONFIG_I:',
|
29
|
+
:junos => 'UI_COMMIT:',
|
30
|
+
}
|
31
|
+
|
32
|
+
class << self
|
33
|
+
def udp port=PORT, listen=0
|
34
|
+
io = UDPSocket.new
|
35
|
+
io.bind listen, port
|
36
|
+
new io, :udp
|
37
|
+
end
|
38
|
+
def file syslog_file=FILE
|
39
|
+
io = open syslog_file, 'r'
|
40
|
+
io.seek 0, IO::SEEK_END
|
41
|
+
new io, :file
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def initialize io, mode=:udp
|
48
|
+
@mode = mode
|
49
|
+
run io
|
50
|
+
end
|
51
|
+
|
52
|
+
def rest opt
|
53
|
+
Oxidized::RestClient.next opt
|
54
|
+
end
|
55
|
+
|
56
|
+
def ios ip, log, i
|
57
|
+
# TODO: we need to fetch 'ip/name' in mode == :file here
|
58
|
+
user = log[i+5]
|
59
|
+
from = log[-1][1..-2]
|
60
|
+
rest( :user => user, :from => from, :model => 'ios', :ip => ip,
|
61
|
+
:name => getname(ip) )
|
62
|
+
end
|
63
|
+
|
64
|
+
def jnpr ip, log, i
|
65
|
+
# TODO: we need to fetch 'ip/name' in mode == :file here
|
66
|
+
user = log[i+2][1..-2]
|
67
|
+
msg = log[(i+6)..-1].join(' ')[10..-2]
|
68
|
+
msg = nil if msg == 'none'
|
69
|
+
rest( :user => user, :msg => msg, :model => 'jnpr', :ip => ip,
|
70
|
+
:name => getname(ip) )
|
71
|
+
end
|
72
|
+
|
73
|
+
def handle_log log, ip
|
74
|
+
log = log.to_s.split ' '
|
75
|
+
if i = log.index(MSG[:ios])
|
76
|
+
ios ip, log, i
|
77
|
+
elsif i = log.index(MSG[:junos])
|
78
|
+
jnpr ip, log, i
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def run io
|
83
|
+
while true
|
84
|
+
log = select [io]
|
85
|
+
log, ip = log.first.first, nil
|
86
|
+
if @mode == :udp
|
87
|
+
log, ip = log.recvfrom_nonblock 2000
|
88
|
+
ip = ip.last
|
89
|
+
else
|
90
|
+
begin
|
91
|
+
log = log.read_nonblock 2000
|
92
|
+
rescue EOFError
|
93
|
+
sleep 1
|
94
|
+
retry
|
95
|
+
end
|
96
|
+
end
|
97
|
+
handle_log log, ip
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def getname ip
|
102
|
+
name = (Resolv.getname ip.to_s rescue ip)
|
103
|
+
NAME_MAP.each { |re, sub| name.sub! re, sub }
|
104
|
+
name
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
Oxidized::SyslogMonitor.udp
|
110
|
+
#Oxidized::SyslogMonitor.file '/var/log/poop'
|
data/lib/oxidized.rb
ADDED
data/lib/oxidized/cli.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
module Oxidized
|
2
|
+
class CLI
|
3
|
+
require 'oxidized'
|
4
|
+
require 'slop'
|
5
|
+
|
6
|
+
def run
|
7
|
+
Process.daemon if @opts[:daemonize]
|
8
|
+
begin
|
9
|
+
Oxidized.new
|
10
|
+
rescue => error
|
11
|
+
crash error
|
12
|
+
raise
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
_args, @opts = parse_opts
|
20
|
+
CFG.debug = true if @opts[:debug]
|
21
|
+
end
|
22
|
+
|
23
|
+
def crash error
|
24
|
+
open Config::Crash, 'w' do |file|
|
25
|
+
file.puts '-' * 50
|
26
|
+
file.puts Time.now.utc
|
27
|
+
file.puts error.message + ' [' + error.class.to_s + ']'
|
28
|
+
file.puts '-' * 50
|
29
|
+
file.puts error.backtrace
|
30
|
+
file.puts '-' * 50
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def parse_opts
|
35
|
+
opts = Slop.new(:help=>true) do
|
36
|
+
on 'd', 'debug', 'turn on debugging'
|
37
|
+
on 'daemonize', 'Daemonize/fork the process'
|
38
|
+
end
|
39
|
+
[opts.parse!, opts]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module Oxidized
|
2
|
+
require 'asetus'
|
3
|
+
class NoConfig < OxidizedError; end
|
4
|
+
class InvalidConfig < OxidizedError; end
|
5
|
+
class Config
|
6
|
+
Root = File.join ENV['HOME'], '.config', 'oxidized'
|
7
|
+
Crash = File.join Root, 'crash'
|
8
|
+
InputDir = File.join Directory, %w(lib oxidized input)
|
9
|
+
OutputDir = File.join Directory, %w(lib oxidized output)
|
10
|
+
ModelDir = File.join Directory, %w(lib oxidized model)
|
11
|
+
SourceDir = File.join Directory, %w(lib oxidized source)
|
12
|
+
Sleep = 1
|
13
|
+
end
|
14
|
+
class << self
|
15
|
+
attr_accessor :mgr
|
16
|
+
end
|
17
|
+
CFGS = Asetus.new :name=>'oxidized', :load=>false, :key_to_s=>true
|
18
|
+
CFGS.default.username = 'username'
|
19
|
+
CFGS.default.password = 'password'
|
20
|
+
CFGS.default.model = 'junos'
|
21
|
+
CFGS.default.interval = 3600
|
22
|
+
CFGS.default.log = File.join Config::Root, 'log'
|
23
|
+
CFGS.default.debug = false
|
24
|
+
CFGS.default.threads = 30
|
25
|
+
CFGS.default.timeout = 20
|
26
|
+
CFGS.default.retries = 3
|
27
|
+
CFGS.default.prompt = /^([\w.@-]+[#>]\s?)$/
|
28
|
+
CFGS.default.rest = '127.0.0.1:8888' # or false to disable
|
29
|
+
CFGS.default.vars = {} # could be 'enable'=>'enablePW'
|
30
|
+
CFGS.default.groups = {} # group level configuration
|
31
|
+
|
32
|
+
CFGS.default.input.default = 'ssh, telnet'
|
33
|
+
CFGS.default.input.ssh.secure = false # complain about changed certs
|
34
|
+
|
35
|
+
CFGS.default.output.default = 'file' # file, git
|
36
|
+
CFGS.default.source.default = 'csv' # csv, sql
|
37
|
+
|
38
|
+
CFGS.default.model_map = {
|
39
|
+
'cisco' => 'ios',
|
40
|
+
'juniper' => 'junos',
|
41
|
+
}
|
42
|
+
|
43
|
+
begin
|
44
|
+
CFGS.load # load system+user configs, merge to Config.cfg
|
45
|
+
rescue => error
|
46
|
+
raise InvalidConfig, "Error loading config: #{error.message}"
|
47
|
+
ensure
|
48
|
+
CFG = CFGS.cfg # convenienence, instead of Config.cfg.password, CFG.password
|
49
|
+
end
|
50
|
+
|
51
|
+
Log.level = Logger::INFO unless CFG.debug
|
52
|
+
raise NoConfig, 'edit ~/.config/oxidized/config' if CFGS.create
|
53
|
+
Log.file = CFG.log if CFG.log
|
54
|
+
|
55
|
+
end
|