oxidized 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e120e1cba4d53d4ddc38c897d735bc48af36cd3d
+  data.tar.gz: e02f9953659fa973777ff3a8120c9d4a28426e0e
+SHA512:
+  metadata.gz: 5f441707347f65ae7e4919ad7ffe392b807484740d59faf88f071c89ec04504ce5320e4211724945bb81c9ee577f239001c32bbf5364b5a6c587e8a5f0393e96
+  data.tar.gz: 8097bad5db23fedf183fa7e2ca17d238e199feb8862d5def0d04d4a3075d901db92729871b9adabdb42a412f5ee4bf4c7804d62a2d6533900b42f2275f23a63d

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+# 0.1.1
+- BUGFIX: vars needs to return value of r, not value of evaluation

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,133 @@
+# Pitch
+ * automatically adds/removes threads to meet configured retrieval interval
+ * restful API to move node immediately to head-of-queue (GET/POST /node/next/[NODE])
+   * syslog udp+file example to catch config change event (ios/junos) and trigger config fetch
+   * will signal ios/junos user who made change, which output module can (git does) use (via POST)
+   * 'git blame' will show for each line who and when the change was made
+ * restful API to reload list of nodes (GET /reload)
+ * restful API to fetch configurations (/node/fetch/[NODE] or /node/fetch/group/[NODE])
+ * restful API to show list of nodes (GET /nodes)
+
+# Install
+ * early days, but try:
+   1. apt-get install ruby ruby-dev libsqlite3-dev libssl-dev
+   2. gem install oxidized
+   3. gem install oxidized-script oxidized-web # if you don't install oxidized-web, make sure you remove "rest" from your config
+   4. oxidized
+   5. vi ~/.config/oxidized/config
+   6. (maybe point to your rancid/router.db or copy it there)
+   7. oxidized
+
+# API
+## Input
+ * gets config from nodes
+ * must implement 'connect', 'get', 'cmd'
+ * 'ssh' and 'telnet' implemented
+
+## Output
+ * stores config
+ * must implement 'store' (may implement 'fetch')
+ * 'git' and 'file' (store as flat ascii) implemented
+
+## Source
+ * gets list of nodes to poll
+ * must implement 'load'
+ * source can have 'name', 'model', 'group', 'username', 'password', 'input', 'output', 'prompt'
+   * name - name of the devices
+   * model - model to use ios/junos/xyz, model is loaded dynamically when needed (Also default in config file)
+   * input - method to acquire config, loaded dynamically as needed (Also default in config file)
+   * output - method to store config, loaded dynamically as needed (Also default in config file)
+   * prompt - prompt used for node (Also default in config file, can be specified in model too)
+ * 'sql' and 'csv' (supports any format with single entry per line, like router.db)
+
+## Model
+ * lists commands to gather from given device model
+ * can use 'cmd', 'prompt', 'comment', 'cfg'
+ * cfg is executed in input/output/source context
+ * cmd is executed in instance of model
+ * 'junos', 'ios', 'ironware' and 'powerconnect' implemented
+
+## Media
+ * TREX 2014 presentation - http://youtu.be/kBQ_CTUuqeU#t=3h
+
+## Cookbook
+
+### Configuration I use in one environment
+```
+---
+username: LANA
+password: LANAAAAAAA
+output:
+  default: git
+  git:
+    user: Oxidized
+    email: o@example.com
+    repo: "/usr/local/lan/oxidized.git"
+source:
+  default: sql
+  sql:
+    adapter: sqlite
+    file: "/usr/local/lan/corona.db"
+    table: device
+    map:
+      name: ptr
+      model: model
+```
+
+### Configuration you end up after first run
+If you don't configure output and source, it'll further fill them with example
+configs for your chosen output/source in subsequent runs
+```
+---
+username: username
+password: password
+model: junos
+interval: 3600
+log: "/home/fisakytt/.config/oxidized/log"
+debug: false
+threads: 30
+timeout: 30
+prompt: !ruby/regexp /^([\w.@-]+[#>]\s?)$/
+rest: 127.0.0.1:8888
+vars: {}
+input:
+  default: ssh, telnet
+  ssh:
+    secure: false
+output:
+  default: git
+source:
+  default: csv
+model_map:
+  cisco: ios
+  juniper: junos
+```
+
+Output and Source could be:
+```
+output:
+  default: git
+  git:
+    user: Oxidized
+    email: o@example.com
+    repo: "/home/fisakytt/.config/oxidized/oxidized.git"
+source:
+  default: csv
+  csv:
+    file: "/home/fisakytt/.config/oxidized/router.db"
+    delimiter: !ruby/regexp /:/
+    map:
+      name: 0
+      model: 1
+```
+which reads nodes from rancid compatible router.db maps their model names to
+model names oxidized expects, stores config in git, will try ssh first then
+telnet, wont crash on changed ssh keys.
+
+Hopefully most of them are obvious, log is ignored if Syslog::Logger exists
+(>=2.0) and syslog is used instead.
+
+System wide configurations can be stored in /etc/oxidized/config, this might be
+useful for storing for example source information, if many users are using
+oxs/Oxidized::Script, which would allow user specific config only to include
+username+password.

data/Rakefile

@@ -0,0 +1,46 @@
+begin
+  require 'rake/testtask'
+  require 'bundler'
+  # Bundler.setup
+rescue LoadError
+  warn 'bunler missing'
+end
+
+gemspec = eval(File.read(Dir['*.gemspec'].first))
+file    = [gemspec.name, gemspec.version].join('-') + '.gem'
+
+desc 'Validate gemspec'
+task :gemspec do
+  gemspec.validate
+end
+
+desc 'Run minitest'
+task :test do
+  Rake::TestTask.new do |t|
+    t.libs.push "lib"
+    t.test_files = FileList['spec/*_spec.rb']
+    t.verbose = true
+  end
+end
+
+desc 'Build gem'
+task :build do
+  system "gem build #{gemspec.name}.gemspec"
+  FileUtils.mkdir_p 'gems'
+  FileUtils.mv file, 'gems'
+end
+
+desc 'Install gem'
+task :install => :build do
+  system "sudo -Es sh -c \'umask 022; gem install gems/#{file}\'"
+end
+
+desc 'Remove gems'
+task :clean do
+  FileUtils.rm_rf 'gems'
+end
+
+desc 'Push to rubygems'
+task :push do
+  system "gem push gems/#{file}"
+end

data/TODO.md

@@ -0,0 +1,20 @@
+# thread number
+  * think about algo
+  * if job ended later than now-iteration have rand(node.size) == 0 to add thread
+  * if now is less than job_ended+iteration same chance to remove thread?
+  * should we try to avoid max threads from being hit? (like maybe non-success thread is pulling average?)
+
+# config
+  * save keys as strings, load as symbols?
+
+# other 
+should it offer cli mass config-pusher? (I think not, I have ideas for such
+program and I'm not sure if synergies are high enough for shared code without
+making both bit awkward)
+
+use sidekiq? Any benefits?
+
+
+# docs, testing
+  * yard docs
+  * rspec tests

data/bin/oxidized

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+
+# FIX ME, killing oxidized needs -9
+trap("INT") { exit } # sinatra will otherwise steal this from us
+
+begin
+  require 'oxidized/cli'
+  Oxidized::CLI.new.run
+rescue => error
+  warn "#{error}"
+  raise if Oxidized::CFG.debug
+end

data/extra/rest_client.rb

@@ -0,0 +1,25 @@
+module Oxidized
+  class RestClient
+    require 'net/http'
+    require 'json'
+    HOST = 'localhost'
+    PORT = 8888
+
+    class << self
+      def next opt={}, host=HOST, port=PORT
+        web = new host, port
+        web.next opt
+      end
+    end
+
+    def initialize host=HOST, port=PORT
+      @web = Net::HTTP.new host, port
+    end
+
+    def next opt
+      data = JSON.dump opt
+      @web.put '/node/next/' + opt[:name].to_s, data
+    end
+
+  end
+end

data/extra/syslog.rb

@@ -0,0 +1,110 @@
+#!/usr/bin/env ruby
+
+# IOS:
+# logging discriminator CFG mnemonics includes CONFIG_I 
+# logging host SERVER discriminator CFG
+
+# JunOS:
+# set system syslog host SERVER interactive-commands notice
+# set system syslog host SERVER match "^mgd\[[0-9]+\]: UI_COMMIT: .*"
+
+# sudo setcap 'cap_net_bind_service=+ep' /usr/bin/ruby
+
+# exit if fork   ## TODO: proper daemonize
+
+require 'socket'
+require 'resolv'
+require './rest_client'
+
+module Oxidized
+  class SyslogMonitor
+    NAME_MAP = {
+      /(.*)\.ip\.tdc\.net/ => '\1',
+      /(.*)\.ip\.fi/       => '\1',
+    }
+    PORT = 514
+    FILE = 'messages'
+    MSG = {
+      :ios   => '%SYS-5-CONFIG_I:',
+      :junos => 'UI_COMMIT:',
+    }
+
+    class << self
+      def udp port=PORT, listen=0
+        io = UDPSocket.new
+        io.bind listen, port
+        new io, :udp
+      end
+      def file syslog_file=FILE
+        io = open syslog_file, 'r'
+        io.seek 0, IO::SEEK_END
+        new io, :file
+      end
+    end
+
+    private 
+
+    def initialize io, mode=:udp
+      @mode = mode
+      run io
+    end
+
+    def rest opt
+      Oxidized::RestClient.next opt
+    end
+
+    def ios ip, log, i
+      # TODO: we need to fetch 'ip/name' in mode == :file here
+      user = log[i+5]
+      from = log[-1][1..-2]
+      rest( :user => user, :from => from, :model => 'ios', :ip => ip,
+            :name => getname(ip) )
+    end
+
+    def jnpr ip, log, i
+      # TODO: we need to fetch 'ip/name' in mode == :file here
+      user = log[i+2][1..-2]
+      msg  = log[(i+6)..-1].join(' ')[10..-2]
+      msg  = nil if msg == 'none'
+      rest( :user => user, :msg => msg, :model => 'jnpr', :ip => ip,
+            :name => getname(ip) )
+    end
+
+    def handle_log log, ip
+      log = log.to_s.split ' '
+      if i = log.index(MSG[:ios])
+        ios ip, log,  i
+      elsif i = log.index(MSG[:junos])
+        jnpr ip, log, i
+      end
+    end
+
+    def run io
+      while true
+        log = select [io]
+        log, ip = log.first.first, nil
+        if @mode == :udp
+          log, ip = log.recvfrom_nonblock 2000
+          ip = ip.last
+        else 
+          begin
+            log = log.read_nonblock 2000
+          rescue EOFError
+            sleep 1
+            retry
+          end
+        end
+        handle_log log, ip
+      end
+    end
+
+    def getname ip
+      name = (Resolv.getname ip.to_s rescue ip)
+      NAME_MAP.each { |re, sub| name.sub! re, sub }
+      name
+    end
+  end
+end
+
+Oxidized::SyslogMonitor.udp
+#Oxidized::SyslogMonitor.file '/var/log/poop'

data/lib/oxidized.rb

@@ -0,0 +1,6 @@
+module Oxidized
+  class OxidizedError < StandardError; end
+  Encoding.default_external = 'UTF-8'
+  Directory = File.expand_path File.join File.dirname(__FILE__), '../' 
+  require 'oxidized/core'
+end

data/lib/oxidized/cli.rb

@@ -0,0 +1,42 @@
+module Oxidized
+  class CLI
+    require 'oxidized'
+    require 'slop'
+
+    def run
+      Process.daemon if @opts[:daemonize]
+      begin
+        Oxidized.new
+      rescue => error
+        crash error
+        raise
+      end
+    end
+
+    private
+
+    def initialize
+      _args, @opts = parse_opts
+      CFG.debug = true if @opts[:debug]
+    end
+
+    def crash error
+      open Config::Crash, 'w' do |file|
+        file.puts '-' * 50
+        file.puts Time.now.utc
+        file.puts error.message + ' [' + error.class.to_s + ']'
+        file.puts '-' * 50
+        file.puts error.backtrace
+        file.puts '-' * 50
+      end
+    end
+
+    def parse_opts
+      opts = Slop.new(:help=>true) do
+        on 'd', 'debug', 'turn on debugging'
+        on 'daemonize',  'Daemonize/fork the process'
+      end
+      [opts.parse!, opts]
+    end
+  end
+end

data/lib/oxidized/config.rb

@@ -0,0 +1,55 @@
+module Oxidized
+  require 'asetus'
+  class NoConfig < OxidizedError; end
+  class InvalidConfig < OxidizedError; end
+  class Config
+    Root      = File.join ENV['HOME'], '.config', 'oxidized'
+    Crash     = File.join Root, 'crash'
+    InputDir  = File.join Directory, %w(lib oxidized input)
+    OutputDir = File.join Directory, %w(lib oxidized output)
+    ModelDir  = File.join Directory, %w(lib oxidized model)
+    SourceDir = File.join Directory, %w(lib oxidized source)
+    Sleep     = 1
+  end
+  class << self
+    attr_accessor :mgr
+  end
+  CFGS = Asetus.new :name=>'oxidized', :load=>false, :key_to_s=>true
+  CFGS.default.username      = 'username'
+  CFGS.default.password      = 'password'
+  CFGS.default.model         = 'junos'
+  CFGS.default.interval      = 3600
+  CFGS.default.log           = File.join Config::Root, 'log'
+  CFGS.default.debug         = false
+  CFGS.default.threads       = 30
+  CFGS.default.timeout       = 20
+  CFGS.default.retries       = 3
+  CFGS.default.prompt        = /^([\w.@-]+[#>]\s?)$/
+  CFGS.default.rest          = '127.0.0.1:8888' # or false to disable
+  CFGS.default.vars          = {}             # could be 'enable'=>'enablePW'
+  CFGS.default.groups        = {}             # group level configuration
+
+  CFGS.default.input.default = 'ssh, telnet'
+  CFGS.default.input.ssh.secure = false # complain about changed certs
+
+  CFGS.default.output.default = 'file'  # file, git
+  CFGS.default.source.default = 'csv'   # csv, sql
+
+  CFGS.default.model_map = {
+    'cisco'   => 'ios',
+    'juniper' => 'junos',
+  }
+
+  begin
+    CFGS.load # load system+user configs, merge to Config.cfg
+  rescue => error
+    raise InvalidConfig, "Error loading config: #{error.message}"
+  ensure
+    CFG = CFGS.cfg  # convenienence, instead of Config.cfg.password, CFG.password
+  end
+
+  Log.level = Logger::INFO unless CFG.debug
+  raise NoConfig, 'edit ~/.config/oxidized/config' if CFGS.create
+  Log.file  = CFG.log if CFG.log
+
+end