oxidized 0.30.1 → 0.32.0

data/examples/podman-compose/Makefile

@@ -6,12 +6,18 @@ help:
 
 rights:
 	podman unshare chown -R 30000:30000 oxidized-config oxidized-ssh
+	podman unshare chown -R 30001 gitserver/repo.git
 
 clean-rights:
 	podman unshare chown -R 0:0 *
 
-start: rights model-image
-	podman-compose -p oxidized up
+start: gitserver-createrepo rights images
+	if [ -f oxidized-config/config ]; then \
+	  podman-compose -p oxidized up ; \
+	else { \
+	  echo "\n########\noxidized-config/config does not exist"; \
+	  echo "create one or copy an example in the folder"; \
+	} fi
 
 run: start
 
@@ -23,7 +29,7 @@ start-local:
 	if [ -f oxidized-config/config.local ]; then \
 	  cp oxidized-config/config.local oxidized-config/config; \
 	else \
-	  echo "oxidized-config/config.local does not exist"; \
+	  echo "\n########\noxidized-config/config.local does not exist"; \
 	fi
 	$(MAKE) start
 
@@ -31,31 +37,67 @@ stop-local: stop
 	if [ -f oxidized-config/config.local ]; then \
 	  git checkout -- oxidized-config/config; \
 	else \
-	  echo "oxidized-config/config.local does not exist"; \
+	  echo "\n########\noxidized-config/config.local does not exist"; \
 	fi
 
+# creates a container image for the model simulation
 model-image:
-	podman image exists localhost/local/model || \
+	podman image exists local/model || \
 	  podman build -t local/model -f model-simulation/Dockerfile-model .
 
 model-clean:
 	podman rmi local/model
 
-clean: stop-local model-clean
+# creates a container image for gitserver
+gitserver-image:
+	podman image exists local/gitserver || \
+	  podman build -t local/gitserver gitserver/
+
+# create the repo repo.git inside the gitserver mapped volume
+gitserver-createrepo: clean-rights
+	if [ ! -d gitserver/repo.git ]; then \
+	  git init --bare gitserver/repo.git; \
+	fi
+
+gitserver-clean:
+	podman rmi local/gitserver
+	rm -rf gitserver/repo.git
+
+gitserver-getkey:
+	podman exec --user oxidized -t oxidized_oxidized_1 sh -c "ssh-keyscan gitserver > /home/oxidized/.ssh/known_hosts"
+
+# build all helper containter images
+images: model-image gitserver-image oxidized-image
+
+# build the oxidized image from the curent repository
+oxidized-image:
+	  podman build -t oxidized:`git describe --tags` -t oxidized:latest ../../
+
+# removes the oxidized image
+oxidized-image-clean:
+	podman rmi local/oxidized
+
+# run evey clean line, even if the previous fails
+clean:
+	-$(MAKE) stop-local
+	-$(MAKE) model-clean
+	-$(MAKE) gitserver-clean
+	-$(MAKE) oxidized-image-clean
 
 define HELP
 make help           - This help
-make rights         - Change the rights of mapped folders for user oxidized
-                      in the container
+make rights         - Change the rights of mapped folders for the users inside
+                      the container
 make clean-rights   - Revert the rights of mapped folders to the local user
-make start          - Start the containter
+make start          - Start the pod with all containers (alias - make run)
                       You can interrupt with Ctrl-C, but make sure you run
-                      make stop to realy stop the container
-make run            - Same as make start
-make stop           - Stop the containter
-make start-local    - Starts the container with the local configuration config.local
-make stop-local     - Stops the container and restores oxidized-config/config from git
-make model-image    - Creates a local OCI-Image to run simulated devices
-make model-clean    - Removes the local OCI-Image to run simulated devices
-make clean          - make stop-local + model-clean
+                      'make stop' to realy stop the container
+make stop           - Stop the pod
+make start-local    - Starts the pod with the local configuration
+                      oxidized-config/config.local
+make stop-local     - Stops the pod and restores
+                      oxidized-config/config from git
+make gitserver-getkey - stores the public key of the gitserver into
+                      oxidized-ssh/known_hosts (the pod must be running)
+make clean          - reverts everything to its original state
 endef

data/examples/podman-compose/README.md

@@ -1,35 +1,47 @@
-# Running oxidized with podman-compose
-This is an example of Oxidized running within an OCI container, provided by
-podman and podman-compose.
+# Running Oxidized with podman-compose
+This example demonstrates running Oxidized within an OCI container using
+podman-compose. It’s actively used in Oxidized development to validate the
+container’s functionality and to simulate potential issues.
 
-In order to have the example work out of the box, a network device is simulated.
-The model asternos has been chosen because there were not too many commands to
-implement.
+While this example uses podman and podman-compose, it should also be compatible
+with docker, as podman supports docker’s CLI.
 
-To run the example, just run `make start`. You should be sure to have installed the
-[dependencies](#dependencies) before.
+To make this example work seamlessly, a simulated network device is included.
+The asternos model is used here for simplicity, as it requires minimal commands
+to implement. The simulated output doesn’t replicate real device responses but
+provides changing lines over time to test Oxidized’s functionality.
 
-To exit, press `CTRL-C` or run `make stop` in a separate shell. If you exit
-with `CTRL-C`, make sure to run `make stop` after it, in order to clean up the
-running environment.
+
+The example also provides a Git server to test the interaction with it.
+
+# Run the example
+> :warning: the example builds local containers and will require at least 2 GB
+> of disk space along with some CPU and time during the first run.
+
+To start the example, simply run `make start`. Ensure you have installed the
+necessary [dependencies](#dependencies) before.
+
+To stop, press `CTRL-C` or run `make stop` in a separate shell. If you exit
+with `CTRL-C`, make sure to run `make stop` afterward to properly clean up the
+environment.
 
 ## Running Environment
-This example of oxidized with podman-compose has been run on Debian
-Bookworm (Version 12), but should work with few adaptations on any Linux
+This example of oxidized with podman-compose is running on Debian
+Bookworm (Version 12). It should work with few adaptations on any Linux
 box running podman, and maybe also with docker.
 
 ## Dependencies
-You need to install some packages on your debian system:
+To get started, install the required packages on your Debian system:
 ```shell
 sudo apt install podman containers-storage podman-compose make
 ```
 
-You also want to make sure that podman uses the overlay driver for storing its images.
-If not, it will save every layer of the container to disk (and not only the delta),
-so it will fill your disk very fast.
+Ensure Podman is using the overlay driver for image storage.
+Without this driver, Podman may save every container layer separately rather
+than only the changes, which can quickly consume disk space.
 
-This happens if you run podman without having installed the package `container-storage`
-before.
+This issue can occur if podman was run before installing the
+`container-storage`  package.
 
 ```shell
 podman info | grep graphDriverName
@@ -43,16 +55,40 @@ You should get this reply
 If not, the quick way I found to solve it is to delete `~/.local/share/containers/`.
 Beware - this will delete **all** your containers!
 
-## I want to adapt this to my needs
-Feel free and have fun. You probably want to edit docker-compose.yml in order to remove the
-simulated model.
+## Adapting to your needs
+Feel free to customize this setup as you wish! You may want to edit
+`docker-compose.yml` to remove any containers simulating specific components.
 
-## Use your own oxidized configuration within the git repository
-When developing oxidized and testing the container, you may want to use your
-own configuration. This can be done by saving it under `oxidized-config/config.local`
+## Use your own oxidized configuration in the git repository
+When developing oxidized or testing the container, you may want to use a custom
+configuration. This can be done by saving it under `oxidized-config/config.local`
 
 `make start-local` will recognize the local configuration and copy it to
 `oxidized-config/config` before starting the container.
 
-You shoud stop the container with `make stop-local` in order to restore the original
-configuration from git.
+You should stop the container with `make stop-local` in order to restore the
+original configuration from the git repository.
+
+In the folder `oxidized-config/, you will also find some example configs,
+for example `config_csv-gitserver`. To use them, just copy the file to `config`.
+
+## Git server public keys
+To enable Oxidized to access the Git server, you'll need to retrieve the
+servers' public SSH keys and store them under `oxidized-ssh/known_hosts`.
+Without this, you will encounter the following error:
+
+```
+ERROR -- : Hook push_to_remote (#<GithubRepo:0x00007f4cff47d918>) failed (#<Rugged::SshError: invalid or unknown remote ssh hostkey>) for event :post_store
+```
+
+While the container environment is running (`make start`), open a separate shell
+and run:
+```
+make gitserver-getkey
+```
+
+You do not need to restart the container environment; Oxidized will
+automatically use the key the next time it pushes to the remote Git repository.
+
+
+

data/examples/podman-compose/docker-compose.yml

@@ -1,8 +1,10 @@
 services:
   oxidized:
     # Choose the image that you want to test
-    # image: docker.io/oxidized/oxidized:0.29.1
-    image: docker.io/oxidized/oxidized:latest
+    # image: docker.io/oxidized/oxidized:0.30.1
+    # image: docker.io/oxidized/oxidized:latest
+    # local/oxidized is build by "make oxidized-image" and "make run"
+    image: local/oxidized
     ports:
       - 127.0.0.1:8042:8888/tcp
     environment:
@@ -13,9 +15,16 @@ services:
     volumes:
        - ./oxidized-config:/home/oxidized/.config/oxidized
        - ./oxidized-ssh:/home/oxidized/.ssh
+
   # This is a simulated network device for the example to work out of the box
   asternos-device:
     image: localhost/local/model
     volumes:
       - ./model-simulation/asternos.sh:/home/oxidized/.profile
       - ./model-simulation/asternos.sh:/home/admin/.profile
+
+  # This is a gitserver to push our configs
+  gitserver:
+    image: localhost/local/gitserver
+    volumes:
+      - ./gitserver/repo.git:/home/git/repo.git

data/examples/podman-compose/gitserver/.gitignore

@@ -0,0 +1 @@
+repo.git

data/examples/podman-compose/gitserver/Dockerfile

@@ -0,0 +1,14 @@
+FROM docker.io/phusion/baseimage:noble-1.0.0
+
+# Use baseimage-docker's init system.
+CMD ["/sbin/my_init"]
+
+# enable ssh
+RUN rm -f /etc/service/sshd/down
+RUN /etc/my_init.d/00_regen_ssh_host_keys.sh
+
+# Add user for the gitserver. The password is "git"
+RUN useradd -m git -p '$6$32WDb0LTFyQkLffy$u15COVx7CQ4tgp4JT4DO4LJ96q/jwFSpuZC3WrllNQDNa6nW1LhJKW9rLV57ak3rj9Ln./aRA85jzeof1B0Gi1' -s /bin/bash -u 30001
+
+# And install git
+RUN install_clean git

data/examples/podman-compose/model-simulation/Dockerfile-model

@@ -1,4 +1,4 @@
-FROM docker.io/phusion/baseimage:jammy-1.0.2
+FROM docker.io/phusion/baseimage:noble-1.0.0
 
 # Use baseimage-docker's init system.
 CMD ["/sbin/my_init"]

data/examples/podman-compose/model-simulation/asternos.sh

@@ -16,6 +16,8 @@ EOF
 function show() {
   if [ "$*" == "version" ]; then
     echo "Version 1.2.3"
+    # Make the output change over time
+    date
   elif [ "$*" == "runningconfiguration all" ]; then
           cat << EOF
 ! begin of the configuration

data/examples/podman-compose/oxidized-config/.gitignore

@@ -3,6 +3,8 @@ config.local
 router.db.local
 
 # Ignore logs, retrieved configs...
+pid
 configs/
 crash
 logs/
+oxidized.git/

data/examples/podman-compose/oxidized-config/config

@@ -10,7 +10,7 @@ use_max_threads: true
 timeout: 20
 retries: 3
 prompt: !ruby/regexp /^([\w.@-]+[#>]\s?)$/
-rest: 127.0.0.1:8888
+rest: 0.0.0.0:8888
 next_adds_job: false
 vars: {}
 groups: {}

data/examples/podman-compose/oxidized-config/config_csv-file

@@ -0,0 +1,46 @@
+---
+username: oxidized
+password: oxidized
+resolve_dns: true
+interval: 3600
+use_syslog: false
+debug: false
+threads: 30
+use_max_threads: true
+timeout: 20
+retries: 3
+prompt: !ruby/regexp /^([\w.@-]+[#>]\s?)$/
+rest: 0.0.0.0:8888
+next_adds_job: false
+vars: {}
+groups: {}
+group_map: {}
+models: {}
+pid: "~/.config/oxidized/pid"
+crash:
+  directory: "~/.config/oxidized/crashes"
+  hostnames: false
+stats:
+  history_size: 10
+input:
+  default: ssh
+  debug: false
+  ssh:
+    secure: false
+  ftp:
+    passive: true
+  utf8_encoded: true
+output:
+  default: file
+  file:
+    directory: "~/.config/oxidized/configs/"
+source:
+  default: csv
+  csv:
+    file: "~/.config/oxidized/router.db"
+    delimiter: !ruby/regexp /:/
+    map:
+      name: 0
+      model: 1
+      ip: 2
+    gpg: false

data/examples/podman-compose/oxidized-config/config_csv-gitserver

@@ -0,0 +1,56 @@
+---
+username: oxidized
+password: oxidized
+resolve_dns: true
+interval: 3600
+use_syslog: false
+debug: false
+threads: 30
+use_max_threads: true
+timeout: 20
+retries: 3
+prompt: !ruby/regexp /^([\w.@-]+[#>]\s?)$/
+rest: 0.0.0.0:8888
+next_adds_job: false
+vars: {}
+groups: {}
+group_map: {}
+models: {}
+pid: "~/.config/oxidized/pid"
+crash:
+  directory: "~/.config/oxidized/crashes"
+  hostnames: false
+stats:
+  history_size: 10
+input:
+  default: ssh
+  debug: false
+  ssh:
+    secure: false
+  ftp:
+    passive: true
+  utf8_encoded: true
+output:
+  default: git
+  git:
+    user: Oxidized
+    email: o@example.com
+    repo: "~/.config/oxidized/oxidized.git"
+source:
+  default: csv
+  csv:
+    file: "~/.config/oxidized/router.db"
+    delimiter: !ruby/regexp /:/
+    map:
+      name: 0
+      model: 1
+      ip: 2
+    gpg: false
+hooks:
+  push_to_remote:
+    type: githubrepo
+    events:
+    - post_store
+    remote_repo: git@gitserver:repo.git
+    username: git
+    password: git

data/examples/podman-compose/oxidized-ssh/.gitignore

@@ -0,0 +1 @@
+known_hosts

data/extra/device2yaml.rb

@@ -0,0 +1,245 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'net/ssh'
+require 'optparse'
+require 'etc'
+require 'timeout'
+
+# This scripts logs in a network device and outputs a yaml file that can be
+# used for model unit tests in spec/model/
+# For more information, see docs/DeviceSimulation.md
+
+# This script is quick & dirty - it grew with the time an could be a project
+# for its own. It works, and that should be enough ;-)
+
+################# Methods
+# Runs cmd in the ssh session, either im exec mode or with a tty
+# saves the output to @output
+def ssh_exec(cmd)
+  puts "\n### Sending #{cmd}..."
+  @output&.puts "  #{cmd}: |-"
+
+  if @exec_mode
+    @ssh_output = @ssh.exec! cmd + "\n"
+  else
+    @ses.send_data cmd + "\n"
+    shell_wait
+  end
+  yaml_output('    ')
+end
+
+# Wait for the ssh command to be executed, with an idle timout @idle_timeout
+# Pressing CTRL-C exits the script
+# Pressing ESC termiates the idle timeout
+def shell_wait
+  @ssh_output = ''
+  # ssh_output gets appended by chanel.on-data (below)
+  # We store the current length of @ssh_output in @ssh_output_length
+  # if @ssh_output.length is bigger than @ssh_output_length, we got new data
+  @ssh_output_length = 0
+
+  # Keep track of time for idle timeout
+  start_time = Time.now
+
+  # Loop & wait for @idle_timeout seconds after last output
+  # 0.1 means that the loop should run at least once per 0.1 second
+  @ssh.loop(0.1) do
+    # if @ssh_output is longer than our saved length, we got new output
+    if @ssh_output_length < @ssh_output.length
+      # reset the timer and save the new output length
+      start_time = Time.now
+      @ssh_output_length = @ssh_output.length
+    end
+
+    # We wait for 0.1 seconds if a key was pressed
+    begin
+      Timeout.timeout(0.1) do
+        # Get input // this is a blocking call
+        char = $stdin.getch
+        # If ctrl-c is pressed, exit the script
+        if char == "\u0003"
+          puts '### CTRL-C pressed, exiting'
+          cleanup
+          exit
+        end
+        # If escape is pressed, terminate idle timeout
+        if char == "\e"
+          puts "\n### ESC pressed, skipping idle timeout"
+          return false
+        else
+          # if not, send the char through ssh
+          @ses.send_data char
+        end
+      end
+    rescue Timeout::Error
+      # No key pressed
+    end
+
+    # exit the loop when the @idle_timeout has been reached (false = exit)
+    Time.now - start_time < @idle_timeout
+  end
+end
+
+def yaml_output(prepend = '')
+  # Now print the collected output to @output
+  firstline = true
+
+  # as we want to prepend 'prepend' to each line, we need each_line and chomp
+  # chomp removes the trainling \n
+  @ssh_output.each_line(chomp: true) do |line|
+    # encode line and remove the first and the trailing double quote
+    line = line.dump[1..-2]
+    if firstline
+      # Make sure the leading space of the first line (if present)
+      # is coded with \0x20 or YAML block scalars won't work
+      line.sub!(/^\A /, '\x20')
+      firstline = false
+    end
+    # Make sure trailing white spaces are coded with \0x20
+    line.gsub!(/ $/, '\x20')
+    # prepend white spaces for the yaml block scalar
+    line = prepend + line
+    @output&.puts line
+  end
+end
+
+def cleanup
+  (@ssh.close rescue true) unless @ssh.closed?
+  @output&.close
+end
+
+################# Main loop
+
+# Define options
+options = {}
+optparse = OptionParser.new do |opts|
+  opts.banner = <<~HEREDOC
+    Usages:
+    - device2yaml.rb [user@]host -i file [options]
+    - device2yaml.rb [user@]host -c "command1
+      command2
+      command3" [options]
+
+    -i and -c are mutualy exclusive, one must be specified
+
+    [options]:
+  HEREDOC
+
+  opts.on('-c', '--commands "command list"', 'specify the commands to be run') do |cmds|
+    options[:commands] = cmds
+  end
+  opts.on('-i', '--input file', 'Specify an input file for commands to be run') do |file|
+    options[:input] = file
+  end
+  opts.on('-o', '--output file', 'Specify an output YAML-file') do |file|
+    options[:output] = file
+  end
+  opts.on('-t', '--timeout value', Integer,
+          'Specify the idle timeout beween commands (default: 5 seconds)') do |timeout|
+    options[:timeout] = timeout
+  end
+  opts.on('-e', '--exec-mode', 'Run ssh in exec mode (without tty)') { @exec_mode = true }
+  opts.on '-h', '--help', 'Print this help' do
+    puts opts
+    exit
+  end
+end
+
+# Catch and parse the first argument
+if ARGV[0] && ARGV[0][0] != '-'
+  argument = ARGV.shift
+  if argument.include?('@')
+    ssh_user, ssh_host = argument.split('@')
+  else
+    ssh_user = Etc.getlogin
+    ssh_host = argument
+  end
+else
+  puts 'Missing a host to connect to...'
+  puts
+  puts optparse
+  exit 1
+end
+
+# Parse the options
+optparse.parse!
+
+# Get the commands to be run against ssh_host
+# ^ = xor = exclusive or
+unless options[:commands].nil? ^ options[:input].nil?
+  puts "Please provide commands to be run against #{ssh_host} with either option -c or -i"
+  puts
+  puts optparse
+  exit 1
+end
+
+if options[:commands]
+  ssh_commands = []
+  options[:commands].each_line(chomp: true) { |command| ssh_commands << command }
+elsif options[:input]
+  ssh_commands = File.read(options[:input]).split(/\n+|\r+/)
+end
+
+puts "Running #{ssh_commands} on #{ssh_user}@#{ssh_host}"
+
+# Defaut idle timeout: 5 seconds, as tests showed that 2 seconds is too short
+@idle_timeout = options[:timeout] || 5
+
+# We will use safe navifation (&.) to call the methods on @output only
+# if @output is not nil
+@output = options[:output] ? File.open(options[:output], 'w') : nil
+
+@ssh = Net::SSH.start(ssh_host,
+                      ssh_user,
+                      { timeout:                         10,
+                        append_all_supported_algorithms: true })
+
+@ssh_output = ''
+
+unless @exec_mode
+  @ses = @ssh.open_channel do |ch|
+    ch.on_data do |_ch, data|
+      @ssh_output += data
+      # Output the data to stdout for interactive control
+      # remove ANSI escape codes, as they can produce problems
+      # The code will be printed as '\e[123m' in the output
+      print data.gsub("\e", '\e')
+    end
+    ch.request_pty(term: 'vt100') do |_ch, success_pty|
+      raise "Can't get PTY" unless success_pty
+
+      ch.send_channel_request 'shell' do |_ch, success_shell|
+        raise "Can't get shell" unless success_shell
+      end
+    end
+    ch.on_extended_data do |_ch, _type, data|
+      $stderr.print "Error: #{data}\n"
+    end
+  end
+end
+
+# YAML begin of file
+@output&.puts '---'
+
+if @exec_mode
+  # init prompt does not exist and is empty in exec mode
+  @output&.puts 'init_prompt:'
+else
+  # get motd and first prompt
+  @output&.puts 'init_prompt: |-'
+  shell_wait
+  yaml_output '  '
+end
+
+@output&.puts 'commands:'
+
+begin
+  ssh_commands.each do |cmd|
+    ssh_exec cmd
+  end
+rescue Errno::ECONNRESET, Net::SSH::Disconnect, IOError => e
+  puts "### Connection closed with message: #{e.message}"
+end
+
+cleanup