oxidized 0.20.0 → 0.28.0

data/Rakefile

@@ -1,8 +1,21 @@
 require 'bundler/gem_tasks'
 require 'rake/testtask'
+require_relative 'lib/oxidized/version'
 
 gemspec = eval(File.read(Dir['*.gemspec'].first))
-file    = [gemspec.name, gemspec.version].join('-') + '.gem'
+gemfile = [gemspec.name, gemspec.version].join('-') + '.gem'
+
+# Integrate Rubocop if available
+begin
+  require 'rubocop/rake_task'
+
+  RuboCop::RakeTask.new
+  task(:default).prerequisites << task(:rubocop)
+rescue LoadError
+  task :rubocop do
+    puts 'Install rubocop to run its rake tasks'
+  end
+end
 
 desc 'Validate gemspec'
 task :gemspec do
@@ -19,10 +32,16 @@ task :test do
   end
 end
 
-## desc 'Install gem'
-## task :install => :build do
-##   system "sudo -Es sh -c \'umask 022; gem install gems/#{file}\'"
-## end
+task build: %i[chmod version_set]
+task :version_set do
+  Oxidized.version_set
+  Bundler::GemHelper.instance.gemspec.version = Oxidized::VERSION
+end
+
+# desc 'Install gem'
+# task install: :build do
+#    system "sudo -Es sh -c \'umask 022; gem install gems/#{gemfile}\'"
+# end
 
 desc 'Remove gems'
 task :clean do
@@ -35,8 +54,30 @@ task :tag do
 end
 
 desc 'Push to rubygems'
-task :push => :tag do
-  system "gem push pkg/#{file}"
+task push: :tag do
+  system "gem push pkg/#{gemfile}"
+end
+
+desc 'Normalise file permissions'
+task :chmod do
+  xbit = %w[
+    bin/oxidized
+    bin/console
+    extra/auto-reload-config.runit
+    extra/nagios_check_failing_nodes.rb
+    extra/oxidized-report-git-commits
+    extra/oxidized.init
+    extra/oxidized.init.d
+    extra/oxidized.runit
+    extra/syslog.rb
+    extra/update-ca-certificates.runit
+  ]
+  dirs = []
+  %x(git ls-files -z).split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }.each do |file|
+    dirs.push(File.dirname(file))
+    xbit.include?(file) ? File.chmod(0o0755, file) : File.chmod(0o0644, file)
+  end
+  dirs.sort.uniq.each { |dir| File.chmod(0o0755, dir) }
 end
 
 task default: :test

data/TODO.md

@@ -1,23 +1,29 @@
-# refactor core
-  * move state from memory to disk, sqlite probably
-  * allows us to retain stats etc over restart
-  * simplifies code
-  * keep only running nodes in memory
-  * negligible I/O cost, as majority is I/O wait getting config
-
-# separate login to owon package
-  * oxidized-script is not only use-case
-  * it would be good to have minimal package used to login to routers
-  * oxidized just one consumer of that functionality
-  * what to do with models, we need model to know how to login. Should models be separated to another package? oxidized-core, oxidized-models and oxidized-login?
-  * how can we allow interactive login in oxidized-login? With functional VTY etc? REPL loop in input/ssh and input/telnet?
-
-# thread number
-  * think about algo
-  * if job ended later than now-iteration have rand(node.size) == 0 to add thread
-  * if now is less than job_ended+iteration same chance to remove thread?
-  * should we try to avoid max threads from being hit? (like maybe non-success thread is pulling average?)
-
-# docs, testing
-  * yard docs
-  * minitest tests
+# To Do
+
+## refactor core
+
+* move state from memory to disk, sqlite probably
+* allows us to retain stats etc over restart
+* simplifies code
+* keep only running nodes in memory
+* negligible I/O cost, as majority is I/O wait getting config
+
+## separate login to own package
+
+* oxidized-script is not only use-case
+* it would be good to have minimal package used to login to routers
+* oxidized just one consumer of that functionality
+* what to do with models, we need model to know how to login. Should models be separated to another package? oxidized-core, oxidized-models and oxidized-login?
+* how can we allow interactive login in oxidized-login? With functional VTY etc? REPL loop in input/ssh and input/telnet?
+
+## thread number
+
+* think about algo
+* if job ended later than now-iteration have rand(node.size) == 0 to add thread
+* if now is less than job_ended+iteration same chance to remove thread?
+* should we try to avoid max threads from being hit? (like maybe non-success thread is pulling average?)
+
+## docs, testing
+
+* yard docs
+* minitest tests

data/bin/console

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-lib = File.expand_path('../../lib', __FILE__)
+lib = File.expand_path('../lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'oxidized'

data/bin/oxidized

@@ -1,14 +1,15 @@
 #!/usr/bin/env ruby
 
-
 # FIX ME, killing oxidized needs -9
 trap("INT") { exit } # sinatra will otherwise steal this from us
 
 begin
-  require 'oxidized/cli'
+  require_relative '../lib/oxidized/cli'
   Oxidized::CLI.new.run
-rescue => error
-  warn "#{error}"
-  debug = Oxidied.config.debug rescue true
+rescue StandardError => error
+  warn error.to_s
+  debug = Oxidized.config.debug rescue true
   raise if debug
+
+  exit 1
 end

data/docs/Configuration.md

@@ -0,0 +1,313 @@
+# Configuration
+
+## Debugging
+
+In case a model plugin doesn't work correctly (ios, procurve, etc.), you can enable live debugging of SSH/Telnet sessions. Just add a `debug` option containing the value true to the `input` section. The log files will be created depending on the parent directory of the logfile option.
+
+The following example will log an active ssh/telnet session `/home/oxidized/.config/oxidized/log/<IP-Address>-<PROTOCOL>`. The file will be truncated on each consecutive ssh/telnet session, so you need to put a `tailf` or `tail -f` on that file!
+
+```yaml
+log: /home/oxidized/.config/oxidized/log
+
+...
+
+input:
+  default: ssh, telnet
+  debug: true
+  ssh:
+    secure: false
+  http:
+    ssl_verify: true
+```
+
+## Privileged mode
+
+To start privileged mode before pulling the configuration, Oxidized needs to send the enable command. You can globally enable this, by adding the following snippet to the global section of the configuration file.
+
+```yaml
+vars:
+   enable: S3cre7
+```
+
+## Removing secrets
+
+To strip out secrets from configurations before storing them, Oxidized needs the `remove_secret` flag. You can globally enable this by adding the following snippet to the global section of the configuration file.
+
+```yaml
+vars:
+  remove_secret: true
+```
+
+Device models that contain substitution filters to remove sensitive data will now be run on any fetched configuration.
+
+As a partial example from ios.rb:
+
+```ruby
+  cmd :secret do |cfg|
+    cfg.gsub! /^(snmp-server community).*/, '\\1 <configuration removed>'
+    (...)
+    cfg
+  end
+```
+
+The above strips out snmp community strings from your saved configs.
+
+**NOTE:** Removing secrets reduces the usefulness as a full configuration backup, but it may make sharing configs easier.
+
+## Disabling SSH exec channels
+
+Oxidized uses exec channels to make information extraction simpler, but there are some situations where this doesn't work well, e.g. configuring devices. This feature can be turned off by setting the `ssh_no_exec`
+variable.
+
+```yaml
+vars:
+  ssh_no_exec: true
+```
+
+## Disabling SSH keepalives
+
+Oxidized SSH input makes use of SSH keepalives to prevent timeouts from slower devices and to quickly tear down stale sessions in larger deployments. There have been reports of SSH keepalives breaking compatibility with certain OS types. They can be disabled using the `ssh_no_keepalive` variable on a per-node basis (by specifying it in the source) or configured application-wide.
+
+```yaml
+vars:
+  ssh_no_keepalive: true
+```
+
+## SSH Auth Methods
+
+By default, Oxidized registers the following auth methods: `none`, `publickey` and `password`. However you can configure this globally, by groups, models or nodes.
+
+```yaml
+vars:
+  auth_methods: [ "none", "publickey", "password", "keyboard-interactive" ]
+```
+
+## Public Key Authentication with SSH
+
+Instead of password-based login, Oxidized can make use of key-based SSH authentication.
+
+You can tell Oxidized to use one or more private keys globally, or specify the key to be used on a per-node basis. The latter can be done by mapping the `ssh_keys` variable through the active source.
+
+Global:
+
+```yaml
+vars:
+  ssh_keys: "~/.ssh/id_rsa"
+```
+
+Per-Node:
+
+```yaml
+...
+map:
+  name: 0
+  model: 1
+vars_map:
+  enable: 2
+  ssh_keys: 3
+...
+```
+
+If you are using a non-standard path, especially when copying the private key via a secured channel, make sure that the permissions are set correctly:
+
+```bash
+foo@bar:~$ ls -la ~/.ssh/
+total 20
+drwx------ 2 oxidized oxidized 4096 Mar 13 17:03 .
+drwx------ 5 oxidized oxidized 4096 Mar 13 21:40 ..
+-r-------- 1 oxidized oxidized  103 Mar 13 17:03 authorized_keys
+-rw------- 1 oxidized oxidized  399 Mar 13 17:02 id_ed25519
+-rw-r--r-- 1 oxidized oxidized   94 Mar 13 17:02 id_ed25519.pub
+```
+
+Finally, multiple private keys can be specified as an array of file paths, such as `["~/.ssh/id_rsa", "~/.ssh/id_another_rsa"]`.
+
+## SSH Proxy Command
+
+Oxidized can `ssh` through a proxy as well. To do so we just need to set `ssh_proxy` variable with the proxy host information and optionally set the `ssh_proxy_port` with the SSH port if it is not listening no port 22.
+
+This can be provided on a per-node basis by mapping the proper fields from your source.
+
+An example for a `csv` input source that maps the 4th field as the `ssh_proxy` value and the 5th field as `ssh_proxy_port`.
+
+```yaml
+...
+map:
+  name: 0
+  model: 1
+vars_map:
+  enable: 2
+  ssh_proxy: 3
+  ssh_proxy_port: 4
+...
+```
+
+## SSH enabling legacy algorithms
+
+When connecting to older firmware over SSH, it is sometimes necessary to enable legacy/disabled settings like KexAlgorithms, HostKeyAlgorithms, MAC or the Encryption.
+
+These settings can be provided on a per-node basis by mapping the ssh_kex, ssh_host_key, ssh_hmac and the ssh_encryption fields from you source.
+
+```yaml
+...
+map:
+  name: 0
+  model: 1
+vars_map:
+  enable: 2
+  ssh_kex: 3
+  ssh_host_key: 4
+  ssh_hmac: 5
+  ssh_encryption: 6
+...
+```
+
+## FTP Passive Mode
+
+Oxidized uses ftp passive mode by default. Some devices require passive mode to be disabled. To do so, we can set `input.ftp.passive` to false - this will make use of FTP active mode.
+
+```yaml
+input:
+  ftp:
+    passive: false
+```
+
+## Advanced Configuration
+
+Below is an advanced example configuration. You will be able to (optionally) override options per device. The router.db format used is `hostname:model:username:password:enable_password`. Hostname and model will be the only required options, all others override the global configuration sections.
+
+```yaml
+---
+username: oxidized
+password: S3cr3tx
+model: junos
+interval: 3600 #interval in seconds
+log: ~/.config/oxidized/log
+debug: false
+threads: 30
+timeout: 20
+retries: 3
+prompt: !ruby/regexp /^([\w.@-]+[#>]\s?)$/
+crash:
+  directory: ~/.config/oxidized/crashes
+  hostnames: false
+vars:
+  enable: S3cr3tx
+groups: {}
+rest: 127.0.0.1:8888
+pid: ~/.config/oxidized/oxidized.pid
+input:
+  default: ssh, telnet
+  debug: false
+  ssh:
+    secure: false
+output:
+  default: git
+  git:
+      user: Oxidized
+      email: oxidized@example.com
+      repo: "~/.config/oxidized/oxidized.git"
+source:
+  default: csv
+  csv:
+    file: ~/.config/oxidized/router.db
+    delimiter: !ruby/regexp /:/
+    map:
+      name: 0
+      model: 1
+      username: 2
+      password: 3
+    vars_map:
+      enable: 4
+model_map:
+  cisco: ios
+  juniper: junos
+```
+
+## Advanced Group Configuration
+
+For group specific credentials
+
+```yaml
+groups:
+  mikrotik:
+    username: admin
+    password: blank
+  ubiquiti:
+    username: ubnt
+    password: ubnt
+```
+
+and add group mapping
+
+```yaml
+map:
+  model: 0
+  name: 1
+  group: 2
+```
+
+For model specific credentials
+
+You can add 'username: nil' if the device only expects a Password at prompt.
+
+```yaml
+models:
+  junos:
+    username: admin
+    password: password
+  ironware:
+    username: admin
+    password: password
+    vars:
+      enable: enablepassword
+  apc_aos:
+    username: apc
+    password: password
+  cisco:
+    username: nil
+    password: pass
+```
+
+## RESTful API and Web Interface
+
+The RESTful API and Web Interface is enabled by configuring the `rest:` parameter in the config file.  This parameter can optionally contain a relative URI.
+
+```yaml
+# Listen on http://127.0.0.1:8888/
+rest: 127.0.0.1:8888
+```
+
+```yaml
+# Listen on http://10.0.0.1:8000/oxidized/
+rest: 10.0.0.1:8000/oxidized
+```
+
+## Triggered backups
+
+A node can be moved to head-of-queue via the REST API `GET/POST /node/next/[NODE]`. This can be useful to immediately schedule a fetch of the configuration after some other event such as a syslog message indicating a configuration update on the device.
+
+In the default configuration this node will be processed when the next job worker becomes available, it could take some time if existing backups are in progress. To execute moved jobs immediately a new job can be added automatically:
+
+```yaml
+next_adds_job: true
+```
+
+This will allow for a more timely fetch of the device configuration.
+
+## Disabling DNS resolution
+
+In some instances it might not be desirable to attempt to resolve names of nodes. One such use case is when nodes are accessed through an SSH proxy, where the remote end resolves the names differently than the host on which Oxidized runs would.
+
+Names can instead be passed verbatim to the input:
+
+```yaml
+resolve_dns: false
+```
+
+## Environment variables
+
+You can use some environment variables to change default root directories values.
+
+* `OXIDIZED_HOME` may be used to set oxidized configuration directory, which defaults to `~/.config/oxidized`
+* `OXIDIZED_LOGS` may be used to set oxidzied logs and crash directories root, which default to `~/.config/oxidized`

data/docs/Creating-Models.md

@@ -0,0 +1,140 @@
+# Creating and Extending Models
+
+Oxidized supports a growing list of [operating system types](Supported-OS-Types.md). Out of the box, most model implementations collect configuration data. Some implementations also include a conservative set of additional commands that collect basic device information (device make and model, software version, licensing information, ...) which are appended to the configuration as comments.
+
+A user may wish to extend an existing model to collect the output of additional commands. Oxidized offers smart loading of models in order to facilitate this with ease, without the need to introduce changes to the upstream source code.
+
+This methodology allows local site changes to be preserved during Oxidized version updates / gem updates. It also enables convenient local development of new models.
+
+## Creating a new model
+
+An Oxidized model, at minimum, requires just three elements:
+
+* A model file, this file should be placed in the ~/.config/oxidized directory and named after the target OS type.
+* A class defined within this file with the same name as the file itself that inherits from `Oxidized::Model`, the base model class.
+* At least one command that will be executed and the output of which will be collected by Oxidized.
+
+A bare-bone example for a fictional model running the OS type `rootware` could be introduced by creating the file `~/.config/oxidized/rootware.rb`, with the following content:
+
+```ruby
+class RootWare < Oxidized::Model
+  
+  cmd 'show complete-config'
+```
+
+This model, as-is will:
+
+* Log into the device and expect the default prompt.
+* Upon matching it, execute the command `show complete-config`
+* Collect the output.
+
+It is often useful to, at minimum, define the following additional elements for any newly introduced module:
+
+* A regexp for the prompt, via the `prompt` statement.
+* A comment prefix, via the `comment` statement.
+* A regexp for telnet username and password prompts.
+* A mechanism for handling `enable` or similar functionality.
+
+The API documentation contains a list of [methods](https://github.com/ytti/oxidized/blob/master/docs/Ruby-API.md#model) that can be used in modules.
+
+A more fleshed out example can be found in the `IOS` and `JunOS` models.
+
+## Extending an existing model with a new command
+
+The example below can be used to extend the `JunOS` model to collect the output of `show interfaces diagnostics optics` and append the output to the configuration file as a comment. This command retrieves DOM information on pluggable optics present in a `JunOS`-powered chassis.
+
+Create the file `~/.config/oxidized/model/junos.rb` with the following contents:
+
+```ruby
+require 'oxidized/model/junos.rb'
+
+
+class JunOS
+
+
+    cmd 'show interfaces diagnostics optics' do |cfg|
+        comment cfg
+    end
+
+
+end
+```
+
+Due to smart loading, the user-supplied `~/.config/oxidized/model/junos.rb` file will take precedence over the model with the same name included in the Oxidized distribution.
+
+The code then uses `require` to initially load the Oxidized-supplied model, and extends the class defined therein with an additional command.
+
+Intuitively, it is also possible to:
+
+* Completely re-define an existing model by creating a file in `~/.config/oxidized/model/` with the same name as an existing model, but not `require`-ing the upstream model file.
+* Create a named variation of an existing model, by creating a file with a new name (such as `~/.config/oxidized/model/junos-extra.rb`), Then `require` the original model and extend its base class as in the above example. The named variation can then be specified as an OS type for specific devices that can benefit from the extra functionality. This allows for preservation of the base functionality for the default model types.
+* Create a completely new model, with a new name, for a new operating system type.
+* Testing/validation of an updated model from the [Oxidized GitHub repo models](https://github.com/ytti/oxidized/tree/master/lib/oxidized/model) by placing an updated model in the proper location without disrupting the gem-supplied model files.
+
+## Advanced features
+
+The loosely-coupled architecture of Oxidized allows for easy extensibility in more advanced use cases as well.
+
+The example below extends the functionality of the `JunOS` model further to collect `display set` formatted configuration from the device, and utilizes the multi-output functionality of the `git` output to place the returned configuration in a separate file within a git repository.
+
+It is possible to configure the `git` output to create new subdirectories under an existing repository instead of creating new repositories for each new defined output type (the default) by including the following configuration in the `~/.config/oxidized/config` file:
+
+```yaml
+output:
+    git:
+        type_as_directory: true
+```
+
+Then, `~/.config/oxidized/model/junos.rb` is adapted as following:
+
+```ruby
+require 'oxidized/model/junos.rb'
+
+
+class JunOS
+
+
+    cmd 'show interface diagnostic optics' do |cfg|
+        comment cfg
+    end
+
+    cmd 'show configuration | display set' do |cfg|
+        cfg.type = "junos-set"
+        cfg.name = "set"
+        cfg
+    end
+
+
+end
+```
+
+The output of the `show configuration | display set` command is marked with a new arbitrary alternative output type, `junos-set`.  The `git` output will use the output type to create a new subdirectory by the same name. In this subdirectory, the `git` output will create files with the name `<devicename>--set` that will contain the output of this command for each device.
+
+## Monkey-patching blocks in existing models
+
+In addition to adding new commands and blocks to existing models, Oxidized offers convenience methods for monkey-patching existing commands and blocks within existing models.
+
+When defining a monkey-patched block, two boolean arguments can be passed as part of the block definition:
+
+* `clear: true`, which resets the existing block, allowing the user to completely override its contents.
+* `prepend: true`, which ensures that the contents of the block are prepended, rather than appended (the default) to an existing block.
+
+This functionality is supported for `cfg`, `cmd`, `pre`, `post`, and `expect` blocks.
+
+Examples:
+
+```ruby
+cmd :secret clear: true do
+  ... "(new code for secret removal which replaces the existing :secret definition in the model)" ...
+end
+```
+
+```ruby
+cmd :ssh do prepend: true do
+  ... "(code that should run first, before any code in the existing :ssh definition in the model)" ...
+end
+```
+
+## Help
+
+If you experience difficulties creating a new model, you can often get help by installing an Internet reachable sanitized device and opening a new issue on Github asking for help with creating the model. You should research what is the device vendor name is, the vendor's OS type name is, the exact device model name and firmware version. This information should be included in the issue.