oxidized 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +2 -0
  3. data/Gemfile +3 -0
  4. data/README.md +133 -0
  5. data/Rakefile +46 -0
  6. data/TODO.md +20 -0
  7. data/bin/oxidized +13 -0
  8. data/extra/rest_client.rb +25 -0
  9. data/extra/syslog.rb +110 -0
  10. data/lib/oxidized.rb +6 -0
  11. data/lib/oxidized/cli.rb +42 -0
  12. data/lib/oxidized/config.rb +55 -0
  13. data/lib/oxidized/config/vars.rb +10 -0
  14. data/lib/oxidized/core.rb +41 -0
  15. data/lib/oxidized/input/cli.rb +48 -0
  16. data/lib/oxidized/input/input.rb +19 -0
  17. data/lib/oxidized/input/ssh.rb +111 -0
  18. data/lib/oxidized/input/telnet.rb +145 -0
  19. data/lib/oxidized/job.rb +14 -0
  20. data/lib/oxidized/jobs.rb +24 -0
  21. data/lib/oxidized/log.rb +21 -0
  22. data/lib/oxidized/manager.rb +57 -0
  23. data/lib/oxidized/model/acos.rb +69 -0
  24. data/lib/oxidized/model/aireos.rb +55 -0
  25. data/lib/oxidized/model/aos.rb +38 -0
  26. data/lib/oxidized/model/aos7.rb +58 -0
  27. data/lib/oxidized/model/aosw.rb +43 -0
  28. data/lib/oxidized/model/eos.rb +32 -0
  29. data/lib/oxidized/model/fortios.rb +44 -0
  30. data/lib/oxidized/model/ios.rb +63 -0
  31. data/lib/oxidized/model/iosxr.rb +47 -0
  32. data/lib/oxidized/model/ironware.rb +33 -0
  33. data/lib/oxidized/model/junos.rb +56 -0
  34. data/lib/oxidized/model/model.rb +152 -0
  35. data/lib/oxidized/model/powerconnect.rb +38 -0
  36. data/lib/oxidized/model/procurve.rb +45 -0
  37. data/lib/oxidized/model/timos.rb +45 -0
  38. data/lib/oxidized/node.rb +169 -0
  39. data/lib/oxidized/node/stats.rb +33 -0
  40. data/lib/oxidized/nodes.rb +143 -0
  41. data/lib/oxidized/output/file.rb +42 -0
  42. data/lib/oxidized/output/git.rb +78 -0
  43. data/lib/oxidized/output/output.rb +5 -0
  44. data/lib/oxidized/source/csv.rb +42 -0
  45. data/lib/oxidized/source/source.rb +11 -0
  46. data/lib/oxidized/source/sql.rb +61 -0
  47. data/lib/oxidized/string.rb +13 -0
  48. data/lib/oxidized/worker.rb +54 -0
  49. data/oxidized.gemspec +20 -0
  50. data/spec/nodes_spec.rb +46 -0
  51. metadata +136 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: e120e1cba4d53d4ddc38c897d735bc48af36cd3d
4
+ data.tar.gz: e02f9953659fa973777ff3a8120c9d4a28426e0e
5
+ SHA512:
6
+ metadata.gz: 5f441707347f65ae7e4919ad7ffe392b807484740d59faf88f071c89ec04504ce5320e4211724945bb81c9ee577f239001c32bbf5364b5a6c587e8a5f0393e96
7
+ data.tar.gz: 8097bad5db23fedf183fa7e2ca17d238e199feb8862d5def0d04d4a3075d901db92729871b9adabdb42a412f5ee4bf4c7804d62a2d6533900b42f2275f23a63d
data/CHANGELOG.md ADDED
@@ -0,0 +1,2 @@
1
+ # 0.1.1
2
+ - BUGFIX: vars needs to return value of r, not value of evaluation
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
data/README.md ADDED
@@ -0,0 +1,133 @@
1
+ # Pitch
2
+ * automatically adds/removes threads to meet configured retrieval interval
3
+ * restful API to move node immediately to head-of-queue (GET/POST /node/next/[NODE])
4
+ * syslog udp+file example to catch config change event (ios/junos) and trigger config fetch
5
+ * will signal ios/junos user who made change, which output module can (git does) use (via POST)
6
+ * 'git blame' will show for each line who and when the change was made
7
+ * restful API to reload list of nodes (GET /reload)
8
+ * restful API to fetch configurations (/node/fetch/[NODE] or /node/fetch/group/[NODE])
9
+ * restful API to show list of nodes (GET /nodes)
10
+
11
+ # Install
12
+ * early days, but try:
13
+ 1. apt-get install ruby ruby-dev libsqlite3-dev libssl-dev
14
+ 2. gem install oxidized
15
+ 3. gem install oxidized-script oxidized-web # if you don't install oxidized-web, make sure you remove "rest" from your config
16
+ 4. oxidized
17
+ 5. vi ~/.config/oxidized/config
18
+ 6. (maybe point to your rancid/router.db or copy it there)
19
+ 7. oxidized
20
+
21
+ # API
22
+ ## Input
23
+ * gets config from nodes
24
+ * must implement 'connect', 'get', 'cmd'
25
+ * 'ssh' and 'telnet' implemented
26
+
27
+ ## Output
28
+ * stores config
29
+ * must implement 'store' (may implement 'fetch')
30
+ * 'git' and 'file' (store as flat ascii) implemented
31
+
32
+ ## Source
33
+ * gets list of nodes to poll
34
+ * must implement 'load'
35
+ * source can have 'name', 'model', 'group', 'username', 'password', 'input', 'output', 'prompt'
36
+ * name - name of the devices
37
+ * model - model to use ios/junos/xyz, model is loaded dynamically when needed (Also default in config file)
38
+ * input - method to acquire config, loaded dynamically as needed (Also default in config file)
39
+ * output - method to store config, loaded dynamically as needed (Also default in config file)
40
+ * prompt - prompt used for node (Also default in config file, can be specified in model too)
41
+ * 'sql' and 'csv' (supports any format with single entry per line, like router.db)
42
+
43
+ ## Model
44
+ * lists commands to gather from given device model
45
+ * can use 'cmd', 'prompt', 'comment', 'cfg'
46
+ * cfg is executed in input/output/source context
47
+ * cmd is executed in instance of model
48
+ * 'junos', 'ios', 'ironware' and 'powerconnect' implemented
49
+
50
+ ## Media
51
+ * TREX 2014 presentation - http://youtu.be/kBQ_CTUuqeU#t=3h
52
+
53
+ ## Cookbook
54
+
55
+ ### Configuration I use in one environment
56
+ ```
57
+ ---
58
+ username: LANA
59
+ password: LANAAAAAAA
60
+ output:
61
+ default: git
62
+ git:
63
+ user: Oxidized
64
+ email: o@example.com
65
+ repo: "/usr/local/lan/oxidized.git"
66
+ source:
67
+ default: sql
68
+ sql:
69
+ adapter: sqlite
70
+ file: "/usr/local/lan/corona.db"
71
+ table: device
72
+ map:
73
+ name: ptr
74
+ model: model
75
+ ```
76
+
77
+ ### Configuration you end up after first run
78
+ If you don't configure output and source, it'll further fill them with example
79
+ configs for your chosen output/source in subsequent runs
80
+ ```
81
+ ---
82
+ username: username
83
+ password: password
84
+ model: junos
85
+ interval: 3600
86
+ log: "/home/fisakytt/.config/oxidized/log"
87
+ debug: false
88
+ threads: 30
89
+ timeout: 30
90
+ prompt: !ruby/regexp /^([\w.@-]+[#>]\s?)$/
91
+ rest: 127.0.0.1:8888
92
+ vars: {}
93
+ input:
94
+ default: ssh, telnet
95
+ ssh:
96
+ secure: false
97
+ output:
98
+ default: git
99
+ source:
100
+ default: csv
101
+ model_map:
102
+ cisco: ios
103
+ juniper: junos
104
+ ```
105
+
106
+ Output and Source could be:
107
+ ```
108
+ output:
109
+ default: git
110
+ git:
111
+ user: Oxidized
112
+ email: o@example.com
113
+ repo: "/home/fisakytt/.config/oxidized/oxidized.git"
114
+ source:
115
+ default: csv
116
+ csv:
117
+ file: "/home/fisakytt/.config/oxidized/router.db"
118
+ delimiter: !ruby/regexp /:/
119
+ map:
120
+ name: 0
121
+ model: 1
122
+ ```
123
+ which reads nodes from rancid compatible router.db maps their model names to
124
+ model names oxidized expects, stores config in git, will try ssh first then
125
+ telnet, wont crash on changed ssh keys.
126
+
127
+ Hopefully most of them are obvious, log is ignored if Syslog::Logger exists
128
+ (>=2.0) and syslog is used instead.
129
+
130
+ System wide configurations can be stored in /etc/oxidized/config, this might be
131
+ useful for storing for example source information, if many users are using
132
+ oxs/Oxidized::Script, which would allow user specific config only to include
133
+ username+password.
data/Rakefile ADDED
@@ -0,0 +1,46 @@
1
+ begin
2
+ require 'rake/testtask'
3
+ require 'bundler'
4
+ # Bundler.setup
5
+ rescue LoadError
6
+ warn 'bunler missing'
7
+ end
8
+
9
+ gemspec = eval(File.read(Dir['*.gemspec'].first))
10
+ file = [gemspec.name, gemspec.version].join('-') + '.gem'
11
+
12
+ desc 'Validate gemspec'
13
+ task :gemspec do
14
+ gemspec.validate
15
+ end
16
+
17
+ desc 'Run minitest'
18
+ task :test do
19
+ Rake::TestTask.new do |t|
20
+ t.libs.push "lib"
21
+ t.test_files = FileList['spec/*_spec.rb']
22
+ t.verbose = true
23
+ end
24
+ end
25
+
26
+ desc 'Build gem'
27
+ task :build do
28
+ system "gem build #{gemspec.name}.gemspec"
29
+ FileUtils.mkdir_p 'gems'
30
+ FileUtils.mv file, 'gems'
31
+ end
32
+
33
+ desc 'Install gem'
34
+ task :install => :build do
35
+ system "sudo -Es sh -c \'umask 022; gem install gems/#{file}\'"
36
+ end
37
+
38
+ desc 'Remove gems'
39
+ task :clean do
40
+ FileUtils.rm_rf 'gems'
41
+ end
42
+
43
+ desc 'Push to rubygems'
44
+ task :push do
45
+ system "gem push gems/#{file}"
46
+ end
data/TODO.md ADDED
@@ -0,0 +1,20 @@
1
+ # thread number
2
+ * think about algo
3
+ * if job ended later than now-iteration have rand(node.size) == 0 to add thread
4
+ * if now is less than job_ended+iteration same chance to remove thread?
5
+ * should we try to avoid max threads from being hit? (like maybe non-success thread is pulling average?)
6
+
7
+ # config
8
+ * save keys as strings, load as symbols?
9
+
10
+ # other
11
+ should it offer cli mass config-pusher? (I think not, I have ideas for such
12
+ program and I'm not sure if synergies are high enough for shared code without
13
+ making both bit awkward)
14
+
15
+ use sidekiq? Any benefits?
16
+
17
+
18
+ # docs, testing
19
+ * yard docs
20
+ * rspec tests
data/bin/oxidized ADDED
@@ -0,0 +1,13 @@
1
+ #!/usr/bin/env ruby
2
+
3
+
4
+ # FIX ME, killing oxidized needs -9
5
+ trap("INT") { exit } # sinatra will otherwise steal this from us
6
+
7
+ begin
8
+ require 'oxidized/cli'
9
+ Oxidized::CLI.new.run
10
+ rescue => error
11
+ warn "#{error}"
12
+ raise if Oxidized::CFG.debug
13
+ end
@@ -0,0 +1,25 @@
1
+ module Oxidized
2
+ class RestClient
3
+ require 'net/http'
4
+ require 'json'
5
+ HOST = 'localhost'
6
+ PORT = 8888
7
+
8
+ class << self
9
+ def next opt={}, host=HOST, port=PORT
10
+ web = new host, port
11
+ web.next opt
12
+ end
13
+ end
14
+
15
+ def initialize host=HOST, port=PORT
16
+ @web = Net::HTTP.new host, port
17
+ end
18
+
19
+ def next opt
20
+ data = JSON.dump opt
21
+ @web.put '/node/next/' + opt[:name].to_s, data
22
+ end
23
+
24
+ end
25
+ end
data/extra/syslog.rb ADDED
@@ -0,0 +1,110 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # IOS:
4
+ # logging discriminator CFG mnemonics includes CONFIG_I
5
+ # logging host SERVER discriminator CFG
6
+
7
+ # JunOS:
8
+ # set system syslog host SERVER interactive-commands notice
9
+ # set system syslog host SERVER match "^mgd\[[0-9]+\]: UI_COMMIT: .*"
10
+
11
+ # sudo setcap 'cap_net_bind_service=+ep' /usr/bin/ruby
12
+
13
+ # exit if fork ## TODO: proper daemonize
14
+
15
+ require 'socket'
16
+ require 'resolv'
17
+ require './rest_client'
18
+
19
+ module Oxidized
20
+ class SyslogMonitor
21
+ NAME_MAP = {
22
+ /(.*)\.ip\.tdc\.net/ => '\1',
23
+ /(.*)\.ip\.fi/ => '\1',
24
+ }
25
+ PORT = 514
26
+ FILE = 'messages'
27
+ MSG = {
28
+ :ios => '%SYS-5-CONFIG_I:',
29
+ :junos => 'UI_COMMIT:',
30
+ }
31
+
32
+ class << self
33
+ def udp port=PORT, listen=0
34
+ io = UDPSocket.new
35
+ io.bind listen, port
36
+ new io, :udp
37
+ end
38
+ def file syslog_file=FILE
39
+ io = open syslog_file, 'r'
40
+ io.seek 0, IO::SEEK_END
41
+ new io, :file
42
+ end
43
+ end
44
+
45
+ private
46
+
47
+ def initialize io, mode=:udp
48
+ @mode = mode
49
+ run io
50
+ end
51
+
52
+ def rest opt
53
+ Oxidized::RestClient.next opt
54
+ end
55
+
56
+ def ios ip, log, i
57
+ # TODO: we need to fetch 'ip/name' in mode == :file here
58
+ user = log[i+5]
59
+ from = log[-1][1..-2]
60
+ rest( :user => user, :from => from, :model => 'ios', :ip => ip,
61
+ :name => getname(ip) )
62
+ end
63
+
64
+ def jnpr ip, log, i
65
+ # TODO: we need to fetch 'ip/name' in mode == :file here
66
+ user = log[i+2][1..-2]
67
+ msg = log[(i+6)..-1].join(' ')[10..-2]
68
+ msg = nil if msg == 'none'
69
+ rest( :user => user, :msg => msg, :model => 'jnpr', :ip => ip,
70
+ :name => getname(ip) )
71
+ end
72
+
73
+ def handle_log log, ip
74
+ log = log.to_s.split ' '
75
+ if i = log.index(MSG[:ios])
76
+ ios ip, log, i
77
+ elsif i = log.index(MSG[:junos])
78
+ jnpr ip, log, i
79
+ end
80
+ end
81
+
82
+ def run io
83
+ while true
84
+ log = select [io]
85
+ log, ip = log.first.first, nil
86
+ if @mode == :udp
87
+ log, ip = log.recvfrom_nonblock 2000
88
+ ip = ip.last
89
+ else
90
+ begin
91
+ log = log.read_nonblock 2000
92
+ rescue EOFError
93
+ sleep 1
94
+ retry
95
+ end
96
+ end
97
+ handle_log log, ip
98
+ end
99
+ end
100
+
101
+ def getname ip
102
+ name = (Resolv.getname ip.to_s rescue ip)
103
+ NAME_MAP.each { |re, sub| name.sub! re, sub }
104
+ name
105
+ end
106
+ end
107
+ end
108
+
109
+ Oxidized::SyslogMonitor.udp
110
+ #Oxidized::SyslogMonitor.file '/var/log/poop'
data/lib/oxidized.rb ADDED
@@ -0,0 +1,6 @@
1
+ module Oxidized
2
+ class OxidizedError < StandardError; end
3
+ Encoding.default_external = 'UTF-8'
4
+ Directory = File.expand_path File.join File.dirname(__FILE__), '../'
5
+ require 'oxidized/core'
6
+ end
@@ -0,0 +1,42 @@
1
+ module Oxidized
2
+ class CLI
3
+ require 'oxidized'
4
+ require 'slop'
5
+
6
+ def run
7
+ Process.daemon if @opts[:daemonize]
8
+ begin
9
+ Oxidized.new
10
+ rescue => error
11
+ crash error
12
+ raise
13
+ end
14
+ end
15
+
16
+ private
17
+
18
+ def initialize
19
+ _args, @opts = parse_opts
20
+ CFG.debug = true if @opts[:debug]
21
+ end
22
+
23
+ def crash error
24
+ open Config::Crash, 'w' do |file|
25
+ file.puts '-' * 50
26
+ file.puts Time.now.utc
27
+ file.puts error.message + ' [' + error.class.to_s + ']'
28
+ file.puts '-' * 50
29
+ file.puts error.backtrace
30
+ file.puts '-' * 50
31
+ end
32
+ end
33
+
34
+ def parse_opts
35
+ opts = Slop.new(:help=>true) do
36
+ on 'd', 'debug', 'turn on debugging'
37
+ on 'daemonize', 'Daemonize/fork the process'
38
+ end
39
+ [opts.parse!, opts]
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,55 @@
1
+ module Oxidized
2
+ require 'asetus'
3
+ class NoConfig < OxidizedError; end
4
+ class InvalidConfig < OxidizedError; end
5
+ class Config
6
+ Root = File.join ENV['HOME'], '.config', 'oxidized'
7
+ Crash = File.join Root, 'crash'
8
+ InputDir = File.join Directory, %w(lib oxidized input)
9
+ OutputDir = File.join Directory, %w(lib oxidized output)
10
+ ModelDir = File.join Directory, %w(lib oxidized model)
11
+ SourceDir = File.join Directory, %w(lib oxidized source)
12
+ Sleep = 1
13
+ end
14
+ class << self
15
+ attr_accessor :mgr
16
+ end
17
+ CFGS = Asetus.new :name=>'oxidized', :load=>false, :key_to_s=>true
18
+ CFGS.default.username = 'username'
19
+ CFGS.default.password = 'password'
20
+ CFGS.default.model = 'junos'
21
+ CFGS.default.interval = 3600
22
+ CFGS.default.log = File.join Config::Root, 'log'
23
+ CFGS.default.debug = false
24
+ CFGS.default.threads = 30
25
+ CFGS.default.timeout = 20
26
+ CFGS.default.retries = 3
27
+ CFGS.default.prompt = /^([\w.@-]+[#>]\s?)$/
28
+ CFGS.default.rest = '127.0.0.1:8888' # or false to disable
29
+ CFGS.default.vars = {} # could be 'enable'=>'enablePW'
30
+ CFGS.default.groups = {} # group level configuration
31
+
32
+ CFGS.default.input.default = 'ssh, telnet'
33
+ CFGS.default.input.ssh.secure = false # complain about changed certs
34
+
35
+ CFGS.default.output.default = 'file' # file, git
36
+ CFGS.default.source.default = 'csv' # csv, sql
37
+
38
+ CFGS.default.model_map = {
39
+ 'cisco' => 'ios',
40
+ 'juniper' => 'junos',
41
+ }
42
+
43
+ begin
44
+ CFGS.load # load system+user configs, merge to Config.cfg
45
+ rescue => error
46
+ raise InvalidConfig, "Error loading config: #{error.message}"
47
+ ensure
48
+ CFG = CFGS.cfg # convenienence, instead of Config.cfg.password, CFG.password
49
+ end
50
+
51
+ Log.level = Logger::INFO unless CFG.debug
52
+ raise NoConfig, 'edit ~/.config/oxidized/config' if CFGS.create
53
+ Log.file = CFG.log if CFG.log
54
+
55
+ end