confctl 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 869735b23809f8bef14f7e1e39b57e7bcaf4a90a37c7e5f4c03064a526cc1c46
-  data.tar.gz: 1badf4f41f9183cbc5c716d436d89b3ccae96519e2f37bf0b67c4ffe7fde89e7
+  metadata.gz: 602260a8486eef31801f7fcf2221f09164e4443e66ea94d6fe534f8f68c56fc2
+  data.tar.gz: 24b4b5dabe240f2e93e0a9784f9eda90bbf52cec2a82fab8e5c5644e323be8da
 SHA512:
-  metadata.gz: 889dc492d0996711bff26a442532cb955dc5fcf5f25c281e25475b7a9e657f64270cc6c839e81b4a3880ab250c50e7a817c30bbfc6c8a37e0b5d7fb755bad212
-  data.tar.gz: b894f7b1720684ad93af743e8fc55f221876c7665d7302682f8354865fdc9c54f17acb990d088981d6638d908624e4687e412dda141eede57556f03bf7f741d7
+  metadata.gz: 87f22fe62a9354b20e711fff429f0a34f6961f9c08489bd42f97ff1d8f9f350bc281e7eca069aecc25081ae71f2ea73f803aa06a0dc4c3d244adbfe10dbbaa86
+  data.tar.gz: 8f44626b3b0e9a97b37171cbc58e94a15b04d87f448110ff211bfa5bc34477d1ce9a0d9f344b19cab93ed8d23f657e44fab3d14c8bad71e62b5d65ea2d3d3d26

data/.editorconfig

@@ -5,7 +5,7 @@ root = true
 charset = utf-8
 end_of_line = lf
 
-[*.{rb,erb,nix}]
+[*.{rb,erb,md,nix}]
 indent_size = 2
 indent_style = space
 trim_trailing_whitespace = true

data/.gitignore

@@ -5,4 +5,5 @@ man/*.html
 man/**/*.html
 man/man?/*.?
 html_doc
+.gems
 .yardoc

data/.rubocop.yml

@@ -1,6 +1,7 @@
 inherit_from: .rubocop_todo.yml
 
 AllCops:
+  TargetRubyVersion: 3.1.0
   NewCops: enable
   Exclude:
     - '*.rb'

data/CHANGELOG.md

@@ -1,2 +1,19 @@
-# Sat Feb 17 2023 -- version 1.0.0
+# Sun Nov 17 2024 -- version 2.0.0
+- Distinguish machine `config` and `metaConfig` (breaking change)
+- Support for machine carriers and netboot servers
+- Optimized git fetch calls when updating software pins
+- Added option `--cores` that is passed to nix-build
+- Added RuboCop
+- Bug fixes
+
+## Transition to `metaConfig`
+The use of `config` has been ambiguous, it could either mean machine
+configuration, i.e. the result of all configured NixOS/vpsAdminOS options,
+or it could mean machine metadata from `module.nix`. Machine metadata
+is now accessible as `metaConfig`.
+
+- `confLib.findConfig` has been renamed to `confLib.findMetaConfig`
+- `confLib.confLib.getClusterMachines` returns a list of machines with `metaConfig` attribute
+
+# Sat Feb 17 2024 -- version 1.0.0
 - Initial release

data/README.md

@@ -13,6 +13,7 @@ machines.
   configurations)
 * Query machine state, view changelogs and diffs
 * Run health checks
+* Support for creating netboot servers, see [docs/carrier.md](docs/carrier.md)
 
 ## Requirements
 
@@ -40,19 +41,12 @@ stored:
 ```
 mkdir cluster-configuration
 ```
-3. Prepare `shell.nix` in the new directory:
-  - Create a `shell.nix` and import the same file from confctl:
+3. Create `shell.nix` and import the same file from confctl:
 ```
 cd cluster-configuration
 cat > shell.nix <<EOF
 import ../confctl/shell.nix
 EOF
-```
-
-  - Alternatively, you can symlink `shell.nix` from the confctl repository:
-```
-cd cluster-configuration
-ln -s ../confctl/shell.nix shell.nix
 ```
 
 4. Enter the `nix-shell`. This will make confctl available and install its
@@ -489,7 +483,7 @@ Then you can use it in machine module as:
 Note that these modules are self-contained. They are not evaluated with the full
 set of NixOS modules. You have to import modules that you need.
 
-## User-defined confctl commands 
+## User-defined confctl commands
 User-defined Ruby scripts can be placed in directory `scripts`. Each script
 should create a subclass of `ConfCtl::UserScript` and call class-method `register`.
 Scripts can define their own `confctl` subcommands.

data/confctl.gemspec

@@ -15,19 +15,19 @@ Gem::Specification.new do |s|
   s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.license     = 'GPL-3.0-only'
 
-  s.required_ruby_version = ">= #{File.read('.ruby-version').strip}"
+  s.required_ruby_version = '>= 3.1.0'
 
-  s.add_runtime_dependency 'curses'
-  s.add_runtime_dependency 'gli', '~> 2.21.0'
-  s.add_runtime_dependency 'json'
-  s.add_runtime_dependency 'md2man'
-  s.add_runtime_dependency 'rainbow', '~> 3.1.1'
-  s.add_runtime_dependency 'rake'
-  s.add_runtime_dependency 'require_all', '~> 2.0.0'
-  s.add_runtime_dependency 'tty-command', '~> 0.10.1'
-  s.add_runtime_dependency 'tty-cursor', '~> 0.7.1'
-  s.add_runtime_dependency 'tty-pager', '~> 0.14.0'
-  s.add_runtime_dependency 'tty-progressbar', '~> 0.18.2'
-  s.add_runtime_dependency 'tty-spinner', '~> 0.9.3'
-  s.add_runtime_dependency 'vpsfree-client', '~> 0.18.0'
+  s.add_dependency 'curses'
+  s.add_dependency 'gli', '~> 2.22.0'
+  s.add_dependency 'json'
+  s.add_dependency 'md2man'
+  s.add_dependency 'rainbow', '~> 3.1.1'
+  s.add_dependency 'rake'
+  s.add_dependency 'require_all', '~> 2.0.0'
+  s.add_dependency 'tty-command', '~> 0.10.1'
+  s.add_dependency 'tty-cursor', '~> 0.7.1'
+  s.add_dependency 'tty-pager', '~> 0.14.0'
+  s.add_dependency 'tty-progressbar', '~> 0.18.2'
+  s.add_dependency 'tty-spinner', '~> 0.9.3'
+  s.add_dependency 'vpsfree-client', '~> 0.19.0'
 end

data/docs/carrier.md

@@ -0,0 +1,138 @@
+# confctl carriers
+Carrier is a machine that can carry other machines. The concept was created
+for the purpose of building netboot servers, but can be used in other settings
+as well.
+
+All machines must be defined within the cluster as usual. We can then designate
+a machine that will serve as a carrier and provide a list of machines that it
+will carry, e.g.:
+
+```nix
+# File cluster/pxe-server/module.nix
+cluster.pxe-server = {
+  # ...
+  carrier = {
+    enable = true;
+
+    # A list of machines found in the cluster/ directory that will be
+    # available on the netboot server. Note that you will have to create
+    # your own buildAttribute, so that the resulting path contains bzImage,
+    # initrd and possibly a machine.json file.
+    machines = [
+      {
+        machine = "node1";
+        buildAttribute = [ "system" "build" "dist" ];
+      }
+    ];
+  };
+};
+```
+
+Now the `node1` machine is available as two machines within the cluster:
+`node1` and `pxe-server#node1`. We can build and deploy both:
+
+  * `confctl deploy node1` will deploy the target machine as defined by the configuration
+  * `confctl deploy pxe-server#node1` will build the machine and copy the result to `pxe-server`, its carrier
+
+We need both commands to build a slightly different output, but based on the same
+configuration. When deploying `node1`, confctl will build attribute
+`config.system.build.toplevel`, where as deploying `pxe-server#node1` will build
+attribute `config.system.build.dist`. This attribute is configured in `buildAttribute`
+option in the example above. `config.system.build.dist` is defined within vpsAdminOS,
+its output is a directory with kernel bzImage, initrd and the root filesystem
+in a squashfs image, i.e. what we need for booting from network.
+
+Custom build attributes can be created by the user. For example, this is how
+`config.system.build.dist` would be defined for NixOS:
+
+```nix
+# File cluster/nixos/config.nix
+{ config, pkgs, lib, confMachine, swpinsInfo, ... }:
+let
+  # machine.json contains metadata about the machine that the carrier uses
+  # to assemble the netboot server
+  machineJson = pkgs.writeText "machine-${config.networking.hostName}.json" (builtins.toJSON {
+    # machine spin, nixos/vpsadminos
+    spin = "nixos";
+
+    # fully qualified domain name
+    fqdn = confMachine.host.fqdn;
+
+    # label used e.g. in user menus
+    label = confMachine.host.fqdn;
+
+    # path to the top-level derivation, needed for system boot
+    toplevel = builtins.unsafeDiscardStringContext config.system.build.toplevel;
+
+    # NixOS version and git revision
+    version = config.system.nixos.version;
+    revision = config.system.nixos.revision;
+
+    # MAC addresses for auto-detection
+    macs = confMachine.netboot.macs;
+
+    # Information used by confctl status
+    swpins-info = swpinsInfo;
+  });
+in {
+  imports = [
+    <nixpkgs/nixos/modules/installer/netboot/netboot-minimal.nix>
+  ];
+
+  # Define custom build attribute
+  system.build.dist = pkgs.symlinkJoin {
+    name = "nixos-netboot";
+    paths = [
+      config.system.build.netbootRamdisk
+      config.system.build.kernel
+      config.system.build.netbootIpxeScript
+    ];
+
+    # Install machine.json
+    postBuild = ''
+      ln -s ${machineJson} $out/machine.json
+    '';
+  };
+
+  # other NixOS configuration
+}
+```
+
+The carrier must be configured to handle the carried machines. Netboot server
+support is integrated within confctl and it only has to be enabled.
+
+```nix
+# File cluster/pxe-server/config.nix
+{ config, ... }:
+{
+  confctl.carrier.netboot = {
+    enable = true;
+
+    # IP address or hostname the netboot server will be available on
+    host = "192.168.100.5";
+
+    # IP ranges that will have access to the server
+    allowedIPv4Ranges = [
+      "192.168.100.0/24"
+    ];
+  };
+}
+```
+
+The netboot server is rebuilt whenever a machine is deployed to it.
+The rebuild can be also run manually using command `build-netboot-server`.
+
+## Other uses
+You can define your own commands to be run on the carrier machine when
+images are deployed to it:
+
+```nix
+# File cluster/pxe-server/config.nix
+{ config, ... }:
+{
+  confctl.carrier.onChangeCommands = ''
+    # List profiles of carried machines
+    ls -l /nix/var/nix/profiles/confctl-*
+  '';
+}
+```

data/lib/confctl/cli/app.rb

@@ -546,6 +546,9 @@ module ConfCtl::Cli
     def nix_build_options(cmd)
       cmd.desc 'Maximum number of build jobs (see nix-build)'
       cmd.flag %w[j max-jobs], arg_name: 'number'
+
+      cmd.desc 'Number of CPU cores to be used (see nix-build)'
+      cmd.flag :cores, arg_name: 'number'
     end
   end
 end

data/lib/confctl/cli/cluster.rb

@@ -110,7 +110,10 @@ module ConfCtl::Cli
     end
 
     def status
-      machines = select_machines(args[0]).managed
+      machines = select_machines(args[0]).managed.select do |_host, machine|
+        machine.target_host || machine.carried?
+      end
+
       raise 'No machines to check' if machines.empty?
 
       ask_confirmation! do
@@ -244,7 +247,7 @@ module ConfCtl::Cli
     end
 
     def test_connection
-      machines = select_machines_with_managed(args[0])
+      machines = select_machines_with_managed(args[0]).runnable
       raise 'No machines to test' if machines.empty?
 
       ask_confirmation! do
@@ -276,7 +279,7 @@ module ConfCtl::Cli
     end
 
     def ssh
-      machines = select_machines_with_managed(args[0])
+      machines = select_machines_with_managed(args[0]).runnable
       raise 'No machines to ssh to' if machines.empty?
 
       if opts['input-string'] && opts['input-file']
@@ -295,7 +298,7 @@ module ConfCtl::Cli
     end
 
     def cssh
-      machines = select_machines_with_managed(args[0])
+      machines = select_machines_with_managed(args[0]).runnable
       raise 'No machines to open cssh to' if machines.empty?
 
       ask_confirmation! do
@@ -508,38 +511,67 @@ module ConfCtl::Cli
         size: :auto,
         reserved_lines: 10
       ) do |lw|
-        if opts['dry-activate-first']
-          lw.sync_console do
-            puts Rainbow(
-              "Trying to activate configuration on #{host} " \
-              "(#{machine.target_host})"
-            ).yellow
-          end
-
-          raise "Error while activating configuration on #{host}" unless nix.activate(machine, toplevel, 'dry-activate')
+        if machine.carried?
+          deploy_carried_to_host(lw, nix, host, machine, toplevel, action)
+        else
+          deploy_standalone_to_host(lw, nix, host, machine, toplevel, action)
         end
 
+        lw.flush
+      end
+    end
+
+    def deploy_standalone_to_host(lw, nix, host, machine, toplevel, action)
+      if opts['dry-activate-first']
         lw.sync_console do
           puts Rainbow(
-            "Activating configuration on #{host} (#{machine.target_host}): " \
-            "#{action}"
+            "Trying to activate configuration on #{host} " \
+            "(#{machine.target_host})"
           ).yellow
         end
 
-        return :skip if opts[:interactive] && !ask_confirmation(always: true)
+        raise "Error while activating configuration on #{host}" unless nix.activate(machine, toplevel, 'dry-activate')
+      end
 
-        raise "Error while activating configuration on #{host}" unless nix.activate(machine, toplevel, action)
+      lw.sync_console do
+        puts Rainbow(
+          "Activating configuration on #{host} (#{machine.target_host}): " \
+          "#{action}"
+        ).yellow
+      end
 
-        if %w[boot switch].include?(action) && !nix.set_profile(machine, toplevel)
-          raise "Error while setting profile on #{host}"
-        end
+      return :skip if opts[:interactive] && !ask_confirmation(always: true)
+
+      # rubocop:disable Style/GuardClause
+
+      unless nix.activate(machine, toplevel, action)
+        raise "Error while activating configuration on #{host}"
+      end
+
+      if %w[boot switch].include?(action) && !nix.set_profile(machine, toplevel)
+        raise "Error while setting profile on #{host}"
       end
+
+      # rubocop:enable Style/GuardClause
+    end
+
+    def deploy_carried_to_host(lw, nix, host, machine, toplevel, action)
+      return if action != 'switch'
+
+      # rubocop:disable Style/GuardClause
+      unless nix.set_carried_profile(machine, toplevel)
+        raise "Error while setting carried profile for #{host} on #{machine.carrier_machine}"
+      end
+      # rubocop:enable Style/GuardClause
     end
 
     def reboot_host(host, machine)
       if machine.localhost?
         puts Rainbow("Skipping reboot of #{host} as it is localhost").yellow
         return :skip
+      elsif machine.carried?
+        puts Rainbow("Skipping reboot of carried machine #{host}").yellow
+        return :skip
       end
 
       puts Rainbow("Rebooting #{host} (#{machine.target_host})").yellow
@@ -761,22 +793,16 @@ module ConfCtl::Cli
       machines.each do |host, machine|
         mc = ConfCtl::MachineControl.new(machine)
 
-        begin
-          puts "#{host}:" unless aggregate
+        puts "#{host}:" unless aggregate
 
-          result = run_ssh_command_on_machine(mc, cmd)
+        result = run_ssh_command_on_machine(mc, cmd)
 
-          if aggregate
-            results[host] = result
-          else
-            puts result.out
-          end
-        rescue TTY::Command::ExitError => e
-          if aggregate
-            results[host] = e
-          else
-            puts e.message
-          end
+        if aggregate
+          results[host] = result
+        elsif result.success?
+          puts result.out
+        else
+          puts result.err
         end
 
         puts unless aggregate
@@ -808,12 +834,8 @@ module ConfCtl::Cli
           tw.add do
             mc = ConfCtl::MachineControl.new(machine)
 
-            begin
-              result = run_ssh_command_on_machine(mc, cmd)
-              results[host] = result
-            rescue TTY::Command::ExitError => e
-              results[host] = e
-            end
+            result = run_ssh_command_on_machine(mc, cmd)
+            results[host] = result
 
             lw.sync_console { pb.advance }
           end
@@ -830,7 +852,13 @@ module ConfCtl::Cli
 
       results.each do |host, result|
         puts "#{host}:"
-        puts result.out
+
+        if result.success?
+          puts result.out
+        else
+          puts result.err
+        end
+
         puts
       end
     end
@@ -844,7 +872,7 @@ module ConfCtl::Cli
         cmd_opts[:in] = opts['input-file']
       end
 
-      mc.execute(*cmd, **cmd_opts)
+      mc.execute!(*cmd, **cmd_opts)
     end
 
     def process_aggregated_results(results)
@@ -903,7 +931,8 @@ module ConfCtl::Cli
     def do_build(machines)
       nix = ConfCtl::Nix.new(
         show_trace: opts['show-trace'],
-        max_jobs: opts['max-jobs']
+        max_jobs: opts['max-jobs'],
+        cores: opts['cores']
       )
       hosts_swpin_paths = {}
 
@@ -996,7 +1025,7 @@ module ConfCtl::Cli
           'Fetching [:bar] :current/:total (:percent)'
         )
 
-        built_generations = nix.build_toplevels(
+        built_generations = nix.build_attributes(
           hosts:,
           swpin_paths:,
           time:,

data/lib/confctl/cli/configuration.rb

@@ -1,8 +1,11 @@
+require_relative '../hook'
 require 'fileutils'
 require 'securerandom'
 
 module ConfCtl::Cli
   class Configuration < Command
+    ConfCtl::Hook.register :configuration_rediscover
+
     DIR_MODE = 0o755
     FILE_MODE = 0o644
 
@@ -238,6 +241,8 @@ module ConfCtl::Cli
 
         f.puts(']')
       end
+
+      ConfCtl::Hook.call(:configuration_rediscover)
     end
 
     protected
@@ -251,8 +256,8 @@ module ConfCtl::Cli
 
         entry_rel_path = File.join(*[rel_path, v].compact)
 
-        if File.exist?(File.join(entry_abs_path, 'module.nix')) \
-           && File.exist?(File.join(entry_abs_path, 'config.nix'))
+        if File.exist?(File.join(entry_abs_path, 'module.nix')) &&
+           File.exist?(File.join(entry_abs_path, 'config.nix'))
           ret << entry_rel_path
         end
 

data/lib/confctl/cli/gen_data.rb

@@ -78,10 +78,28 @@ module ConfCtl::Cli
         q.echo = false
       end.to_s
 
-      @vpsadmin_client.authenticate(:basic, user:, password:)
+      @vpsadmin_client.authenticate(:token, user:, password:, lifetime: 'fixed', interval: 60) do |_action, params|
+        ret = {}
+
+        params.each do |name, desc|
+          ret[name] = read_auth_param(name, desc)
+        end
+
+        ret
+      end
+
       @vpsadmin_client
     end
 
+    def read_auth_param(name, p)
+      prompt = "#{p[:label] || name}: "
+
+      ask(prompt) do |q|
+        q.default = nil
+        q.echo = !p[:protected]
+      end
+    end
+
     def update_file(relpath, &)
       abs = File.join(data_dir, relpath)
       tmp = "#{abs}.new"

data/lib/confctl/cli/generation.rb

@@ -31,7 +31,7 @@ module ConfCtl::Cli
 
       return unless opts[:remote] && opts[:gc]
 
-      machines_gc = machines.select do |host, _machine|
+      machines_gc = machines.runnable.select do |host, _machine|
         gens.detect { |gen| gen.host == host }
       end
 
@@ -68,7 +68,7 @@ module ConfCtl::Cli
 
       global = ConfCtl::Settings.instance.host_generations
 
-      machines_gc = machines.select do |_host, machine|
+      machines_gc = machines.runnable.select do |_host, machine|
         gc = machine['buildGenerations']['collectGarbage']
 
         if gc.nil?
@@ -82,7 +82,7 @@ module ConfCtl::Cli
     end
 
     def collect_garbage
-      machines = select_machines(args[0])
+      machines = select_machines(args[0]).runnable
 
       raise 'No machines to collect garbage on' if machines.empty?
 
@@ -295,6 +295,8 @@ module ConfCtl::Cli
         retvals = executor.run
         failed = retvals.compact
 
+        lw.flush
+
         raise "Gargabe collection failed on: #{failed.join(', ')}" if failed.any?
       end
     end

data/lib/confctl/generation/host_list.rb

@@ -1,16 +1,16 @@
 module ConfCtl
   class Generation::HostList
     # @param mc [MachineControl]
+    # @param profile [String]
     # @return [Generation::HostList]
-    def self.fetch(mc, profile: '/nix/var/nix/profiles/system')
-      out, = mc.bash_script(<<-END
+    def self.fetch(mc, profile:)
+      out, = mc.bash_script(<<~END)
         realpath #{profile}
 
         for generation in `ls -d -1 #{profile}-*-link` ; do
           echo "$generation;$(readlink $generation);$(stat --format=%Y $generation)"
         done
       END
-                           )
 
       list = new(mc.machine.name)
       lines = out.strip.split("\n")

data/lib/confctl/git_repo_mirror.rb

@@ -14,13 +14,13 @@ module ConfCtl
       @cmd = SystemCommand.new
     end
 
-    def setup
+    def setup(ref: nil)
       File.stat(mirror_path)
     rescue Errno::ENOENT
       FileUtils.mkdir_p(mirror_path)
       git('clone', args: ['--mirror', url, mirror_path])
     else
-      git_repo('fetch')
+      git_repo('fetch', opts: ['--no-show-forced-updates'], args: (ref ? ['origin', ref] : []))
     end
 
     def revision_parse(str)