confctl 2.0.0 → 2.1.0

data/lib/confctl/generation/build.rb

@@ -16,6 +16,9 @@ module ConfCtl
     # @return [String]
     attr_reader :toplevel
 
+    # @return [String]
+    attr_reader :auto_rollback
+
     # @return [Array<String>]
     attr_reader :swpin_names
 
@@ -29,22 +32,28 @@ module ConfCtl
     # @return [Boolean]
     attr_accessor :current
 
+    # @return [String, nil]
+    attr_reader :kernel_version
+
     # @param host [String]
     def initialize(host)
       @host = host
     end
 
     # @param toplevel [String]
+    # @param auto_rollback [String]
     # @param swpin_paths [Hash]
     # @param swpin_specs [Hash]
     # @param date [Time]
-    def create(toplevel, swpin_paths, swpin_specs, date: nil)
+    def create(toplevel, auto_rollback, swpin_paths, swpin_specs, date: nil)
       @toplevel = toplevel
+      @auto_rollback = auto_rollback
       @swpin_names = swpin_paths.keys
       @swpin_paths = swpin_paths
       @swpin_specs = swpin_specs
       @date = date || Time.now
       @name = date.strftime('%Y-%m-%d--%H-%M-%S')
+      @kernel_version = extract_kernel_version
     end
 
     # @param name [String]
@@ -53,6 +62,7 @@ module ConfCtl
 
       cfg = JSON.parse(File.read(config_path))
       @toplevel = cfg['toplevel']
+      @auto_rollback = cfg['auto_rollback']
 
       @swpin_names = []
       @swpin_paths = {}
@@ -69,6 +79,7 @@ module ConfCtl
       end
 
       @date = Time.iso8601(cfg['date'])
+      @kernel_version = extract_kernel_version
     rescue StandardError => e
       raise Error, "invalid generation '#{name}': #{e.message}"
     end
@@ -76,6 +87,7 @@ module ConfCtl
     def save
       FileUtils.mkdir_p(dir)
       File.symlink(toplevel, toplevel_path)
+      File.symlink(auto_rollback, auto_rollback_path)
 
       swpin_paths.each do |name, path|
         File.symlink(path, swpin_path(name))
@@ -85,6 +97,7 @@ module ConfCtl
         f.puts(JSON.pretty_generate({
           date: date.iso8601,
           toplevel:,
+          auto_rollback:,
           swpins: swpin_paths.to_h do |name, path|
             [name, { path:, spec: swpin_specs[name].as_json }]
           end
@@ -97,6 +110,13 @@ module ConfCtl
     def destroy
       remove_gcroot
       File.unlink(toplevel_path)
+
+      begin
+        File.unlink(auto_rollback_path)
+      rescue Errno::ENOENT
+        # Older generations might not have auto_rollback
+      end
+
       swpin_paths.each_key { |name| File.unlink(swpin_path(name)) }
       File.unlink(config_path)
       Dir.rmdir(dir)
@@ -104,6 +124,7 @@ module ConfCtl
 
     def add_gcroot
       GCRoot.add(gcroot_name('toplevel'), toplevel_path)
+      GCRoot.add(gcroot_name('auto_rollback'), auto_rollback_path)
       swpin_paths.each_key do |name|
         GCRoot.add(gcroot_name("swpin.#{name}"), toplevel_path)
       end
@@ -111,6 +132,7 @@ module ConfCtl
 
     def remove_gcroot
       GCRoot.remove(gcroot_name('toplevel'))
+      GCRoot.remove(gcroot_name('auto_rollback'))
       swpin_paths.each_key do |name|
         GCRoot.remove(gcroot_name("swpin.#{name}"))
       end
@@ -130,6 +152,10 @@ module ConfCtl
       @toplevel_path ||= File.join(dir, 'toplevel')
     end
 
+    def auto_rollback_path
+      @auto_rollback_path ||= File.join(dir, 'auto_rollback')
+    end
+
     def swpin_path(name)
       File.join(dir, "#{name}.swpin")
     end
@@ -141,5 +167,20 @@ module ConfCtl
     def gcroot_name(file)
       "#{escaped_host}-generation-#{name}-#{file}"
     end
+
+    def extract_kernel_version
+      # `kernel` is for NixOS/vpsAdminOS and also carried NixOS machines (netboot)
+      # `bzImage` is for carried vpsAdminOS machines (netboot)
+      %w[kernel bzImage].each do |v|
+        link = File.readlink(File.join(toplevel, v))
+        next unless %r{\A/nix/store/[^-]+-linux-([^/]+)} =~ link
+
+        return ::Regexp.last_match(1)
+      rescue Errno::ENOENT, Errno::EINVAL
+        next
+      end
+
+      nil
+    end
   end
 end

data/lib/confctl/generation/build_list.rb

@@ -55,6 +55,16 @@ module ConfCtl
       index[name]
     end
 
+    # @param offset [Integer] 0 = current/last, 1 = first (oldest), -1 = before last
+    # @return [Generation::Build]
+    def at_offset(offset)
+      if offset == 0
+        generations.last
+      else
+        generations[offset - 1]
+      end
+    end
+
     def each(&)
       generations.each(&)
     end

data/lib/confctl/generation/host.rb

@@ -1,19 +1,22 @@
 module ConfCtl
   class Generation::Host
-    attr_reader :host, :profile, :id, :toplevel, :date, :current
+    attr_reader :host, :profile, :id, :toplevel, :date, :kernel_version, :current
 
-    # @param host [String]
+    # @param machine [Machine]
     # @param profile [String]
     # @param id [Integer]
     # @param toplevel [String]
     # @param date [Time]
+    # @param kernel_version [String, nil]
     # @param mc [MachineControl]
-    def initialize(host, profile, id, toplevel, date, current: false, mc: nil)
-      @host = host
+    def initialize(machine, profile, id, toplevel, date, kernel_version, current: false, mc: nil)
+      @host = machine.name
+      @machine = machine
       @profile = profile
       @id = id
       @toplevel = toplevel
       @date = date
+      @kernel_version = kernel_version
       @current = current
       @mc = mc
     end
@@ -25,7 +28,8 @@ module ConfCtl
     def destroy
       raise 'machine control not available' if mc.nil?
 
-      mc.execute('nix-env', '-p', profile, '--delete-generations', id.to_s)
+      env_cmd = @machine.carried? ? 'carrier-env' : 'nix-env'
+      mc.execute(env_cmd, '-p', profile, '--delete-generations', id.to_s)
     end
 
     protected

data/lib/confctl/generation/host_list.rb

@@ -1,24 +1,33 @@
 module ConfCtl
   class Generation::HostList
+    # @parma machine [Machine]
     # @param mc [MachineControl]
     # @param profile [String]
     # @return [Generation::HostList]
-    def self.fetch(mc, profile:)
+    def self.fetch(machine, mc, profile:)
       out, = mc.bash_script(<<~END)
         realpath #{profile}
 
         for generation in `ls -d -1 #{profile}-*-link` ; do
-          echo "$generation;$(readlink $generation);$(stat --format=%Y $generation)"
+          echo -n "$generation;"
+          echo -n "$(readlink $generation);"
+          echo -n "$(stat --format=%Y $generation);"
+
+          for kernel_file in kernel bzImage ; do
+            [ -h "$generation/$kernel_file" ] && echo -n $(readlink "$generation/$kernel_file")
+          done
+
+          echo
         done
       END
 
-      list = new(mc.machine.name)
+      list = new(machine.name)
       lines = out.strip.split("\n")
       current_path = lines.shift
       id_rx = /^#{Regexp.escape(profile)}-(\d+)-link$/
 
       lines.each do |line|
-        link, path, created_at = line.split(';')
+        link, path, created_at, kernel = line.split(';')
 
         if id_rx =~ link
           id = ::Regexp.last_match(1).to_i
@@ -27,12 +36,18 @@ module ConfCtl
           next
         end
 
+        kernel_version =
+          if kernel && %r{\A/nix/store/[^-]+-linux-([^/]+)} =~ kernel
+            ::Regexp.last_match(1)
+          end
+
         list << Generation::Host.new(
-          mc.machine.name,
+          machine,
           profile,
           id,
           path,
           Time.at(created_at.to_i),
+          kernel_version,
           current: path == current_path,
           mc:
         )

data/lib/confctl/generation/unified.rb

@@ -15,6 +15,9 @@ module ConfCtl
     # @return [Time]
     attr_reader :date
 
+    # @return [String, nil]
+    attr_reader :kernel_version
+
     # @return [Boolean]
     attr_reader :current
 
@@ -37,11 +40,13 @@ module ConfCtl
         @name = build_generation.name
         @toplevel = build_generation.toplevel
         @date = build_generation.date
+        @kernel_version = build_generation.kernel_version
         @current ||= build_generation.current
       elsif host_generation
         @name = host_generation.approx_name
         @toplevel = host_generation.toplevel
         @date = host_generation.date
+        @kernel_version = host_generation.kernel_version
         @current ||= host_generation.current
       else
         raise ArgumentError, 'set build or host'

data/lib/confctl/generation/unified_list.rb

@@ -46,6 +46,16 @@ module ConfCtl
       generations.each(&)
     end
 
+    # @param offset [Integer] 0 = current/last, 1 = first (oldest), -1 = before last
+    # @return [Generation::Unified]
+    def at_offset(offset)
+      if offset == 0
+        generations.last
+      else
+        generations[offset - 1]
+      end
+    end
+
     def delete_if(&)
       generations.delete_if(&)
     end

data/lib/confctl/machine.rb

@@ -116,6 +116,10 @@ module ConfCtl
       end
     end
 
+    def auto_rollback?
+      meta.fetch('autoRollback', {}).fetch('enable', true)
+    end
+
     def health_checks
       return @health_checks if @health_checks
 

data/lib/confctl/machine_control.rb

@@ -6,10 +6,11 @@ module ConfCtl
     attr_reader :machine
 
     # @param machine [Machine]
-    def initialize(machine)
+    # @param logger [#<<]
+    def initialize(machine, logger: nil)
       @machine = machine
       @extra_ssh_opts = []
-      @cmd = SystemCommand.new
+      @cmd = SystemCommand.new(logger:)
     end
 
     # Try to open SSH connection

data/lib/confctl/machine_list.rb

@@ -90,6 +90,10 @@ module ConfCtl
       !empty?
     end
 
+    def to_a
+      @machines.values
+    end
+
     # @return [Array<HealthChecks::Base>]
     def health_checks
       checks = []

data/lib/confctl/machine_status.rb

@@ -103,7 +103,7 @@ module ConfCtl
 
       if generations
         begin
-          @generations = Generation::HostList.fetch(mc, profile: machine.profile)
+          @generations = Generation::HostList.fetch(machine, mc, profile: machine.profile)
         rescue TTY::Command::ExitError
           return
         end

data/lib/confctl/nix.rb

@@ -1,4 +1,5 @@
 require 'confctl/utils/file'
+require 'digest'
 require 'etc'
 require 'json'
 require 'securerandom'
@@ -137,19 +138,19 @@ module ConfCtl
       end
     end
 
-    # Evaluate swpins for host
-    # @param host [String]
-    # @return [Hash]
-    def eval_host_swpins(host)
+    # Evaluate swpins for hosts
+    # @param hosts [Array<String>]
+    # @return [Hash] host => swpins
+    def eval_host_swpins(hosts)
       with_argument({
         confDir: conf_dir,
         build: :evalHostSwpins,
-        machines: [host]
+        machines: hosts
       }, core_swpins: true) do |arg|
         out_link = File.join(
           cache_dir,
           'build',
-          "#{ConfCtl.safe_host_name(host)}.swpins"
+          "#{Digest::SHA256.hexdigest(hosts.join(','))[0..11]}.swpins"
         )
 
         cmd_args = [
@@ -164,7 +165,7 @@ module ConfCtl
 
         out, = cmd.run(*cmd_args)
 
-        JSON.parse(File.read(out.strip))[host]
+        JSON.parse(File.read(out.strip))
       end
     end
 
@@ -204,15 +205,16 @@ module ConfCtl
         nb.run(&block)
 
         begin
-          host_toplevels = JSON.parse(File.read(out_link))
-          host_toplevels.each do |host, toplevel|
+          host_results = JSON.parse(File.read(out_link))
+          host_results.each do |host, result|
             host_generations = Generation::BuildList.new(host)
-            generation = host_generations.find(toplevel, swpin_paths)
+            generation = host_generations.find(result['attribute'], swpin_paths)
 
             if generation.nil?
               generation = Generation::Build.new(host)
               generation.create(
-                toplevel,
+                result['attribute'],
+                result['autoRollback'],
                 swpin_paths,
                 host_swpin_specs[host],
                 date: time
@@ -232,35 +234,78 @@ module ConfCtl
     end
 
     # @param machine [Machine]
-    # @param toplevel [String]
+    # @param paths [Array<String>]
     #
     # @yieldparam progress [Integer]
     # @yieldparam total [Integer]
     # @yieldparam path [String]
     #
     # @return [Boolean]
-    def copy(machine, toplevel, &)
+    def copy(machine, paths, &)
       if machine.localhost?
         true
       elsif machine.carried?
-        cp = NixCopy.new(machine.carrier_machine.target_host, toplevel)
+        cp = NixCopy.new(machine.carrier_machine.target_host, paths)
         cp.run!(&).success?
       else
-        cp = NixCopy.new(machine.target_host, toplevel)
+        cp = NixCopy.new(machine.target_host, paths)
         cp.run!(&).success?
       end
     end
 
     # @param machine [Machine]
-    # @param toplevel [String]
+    # @param generation [Generation::Build]
     # @param action [String]
     # @return [Boolean]
-    def activate(machine, toplevel, action)
-      args = [File.join(toplevel, 'bin/switch-to-configuration'), action]
+    def activate(machine, generation, action)
+      args = [File.join(generation.toplevel, 'bin/switch-to-configuration'), action]
 
       MachineControl.new(machine).execute!(*args).success?
     end
 
+    # @param machine [Machine]
+    # @param generation [Generation::Build]
+    # @param action [String]
+    # @return [Boolean]
+    def activate_with_rollback(machine, generation, action)
+      check_file = File.join('/run', "confctl-confirm-#{SecureRandom.hex(3)}")
+      timeout = machine['autoRollback']['timeout']
+      logger = NullLogger.new
+
+      args = [
+        generation.auto_rollback,
+        '-t', timeout,
+        generation.toplevel,
+        action,
+        check_file
+      ]
+
+      activation_success = nil
+
+      activation_thread = Thread.new do
+        activation_success = MachineControl.new(machine).execute!(*args).success?
+      end
+
+      # Wait for the configuration to be switched
+      t = Time.now
+
+      loop do
+        out, = MachineControl.new(machine, logger:).execute!('cat', check_file, '2>/dev/null')
+        stripped = out.strip
+        break if stripped == 'switched' || ((t + timeout + 10) < Time.now && stripped != 'switching')
+
+        sleep(1)
+      end
+
+      # Confirm it
+      10.times do
+        break if MachineControl.new(machine, logger:).execute!('sh', '-c', "'echo confirmed > #{check_file}'").success?
+      end
+
+      activation_thread.join
+      activation_success
+    end
+
     # @param machine [Machine]
     # @param toplevel [String]
     # @return [Boolean]

data/lib/confctl/nix_copy.rb

@@ -1,10 +1,10 @@
 module ConfCtl
   class NixCopy
     # @param target [String]
-    # @param store_path [String]
-    def initialize(target, store_path)
+    # @param store_paths [Array<String>]
+    def initialize(target, store_paths)
       @target = target
-      @store_path = store_path
+      @store_paths = store_paths
       @total = nil
       @progress = 0
     end
@@ -22,7 +22,7 @@ module ConfCtl
       ret = cmd.run!(
         'nix-copy-closure',
         '--to', "root@#{target}",
-        store_path,
+        *store_paths,
         &line_buf.feed_block
       )
 
@@ -32,7 +32,7 @@ module ConfCtl
 
     protected
 
-    attr_reader :target, :store_path
+    attr_reader :target, :store_paths
 
     def parse_line(line)
       if @total.nil? && /^copying (\d+) paths/ =~ line

data/lib/confctl/null_logger.rb

@@ -0,0 +1,7 @@
+module ConfCtl
+  class NullLogger
+    def <<(_str)
+      # do nothing
+    end
+  end
+end

data/lib/confctl/swpins/specs/git_rev.rb

@@ -5,7 +5,7 @@ module ConfCtl
     handle :'git-rev'
 
     def prefetch_set(args)
-      super(args)
+      super
       wrap_fetcher
     end
 

data/lib/confctl/system_command.rb

@@ -2,9 +2,10 @@ require 'tty-command'
 
 module ConfCtl
   module SystemCommand
+    # @param logger [#<<]
     # @return [TTY::Command]
-    def self.new
-      TTY::Command.new(output: Logger.instance, color: false)
+    def self.new(logger: nil)
+      TTY::Command.new(output: logger || Logger.instance, color: false)
     end
   end
 end

data/lib/confctl/version.rb

@@ -1,3 +1,3 @@
 module ConfCtl
-  VERSION = '2.0.0'.freeze
+  VERSION = '2.1.0'.freeze
 end

data/libexec/auto-rollback.rb

@@ -0,0 +1,106 @@
+#!@ruby@/bin/ruby
+# Switch to a new system configuration and wait for confctl to confirm
+# connectivity. If confctl is unable to reach the deployed machine, this
+# script will roll back to the previous configuration.
+
+require 'optparse'
+
+class AutoRollback
+  def self.run
+    ar = new
+    ar.run
+  end
+
+  def run
+    puts 'Deploying with auto-rollback'
+
+    options = parse_options
+
+    current_system = File.readlink('/run/current-system')
+    puts "  current system = #{current_system}"
+    puts "  new system     = #{options[:toplevel]}"
+    puts "  action         = #{options[:action]}"
+    puts "  check file     = #{options[:check_file]}"
+    puts "  timeout        = #{options[:timeout]} seconds"
+    puts
+
+    puts 'Switching to new configuration'
+    File.write(options[:check_file], 'switching')
+
+    pid = Process.spawn(
+      File.join(options[:toplevel], 'bin/switch-to-configuration'),
+      options[:action]
+    )
+
+    Process.wait(pid)
+
+    File.write(options[:check_file], 'switched')
+
+    puts 'Switch complete, waiting for confirmation'
+    t = Time.now
+
+    loop do
+      sleep(0.5)
+
+      if File.read(options[:check_file]).strip == 'confirmed'
+        puts 'Configuration confirmed'
+        File.unlink(options[:check_file])
+        exit
+      end
+
+      break if t + options[:timeout] < Time.now
+    end
+
+    puts 'Timeout occurred, rolling back'
+
+    pid = Process.spawn(
+      File.join(current_system, 'bin/switch-to-configuration'),
+      options[:action]
+    )
+
+    Process.wait(pid)
+
+    puts 'Rollback complete'
+    exit(false)
+  end
+
+  protected
+
+  def parse_options
+    options = {
+      timeout: 60,
+      toplevel: nil,
+      action: nil,
+      check_file: nil
+    }
+
+    opt_parser = OptionParser.new do |parser|
+      parser.banner = "Usage: #{$0} [options] <toplevel> <action> <check file>"
+
+      parser.on('-t', '--timeout TIMEOUT', Integer, 'Timeout in seconds') do |v|
+        options[:timeout] = v
+      end
+
+      parser.on('-h', '--help', 'Print help message and exit') do
+        puts parser
+        exit
+      end
+    end
+
+    opt_parser.parse!
+
+    if ARGV.length != 3
+      warn 'Invalid arguments'
+      warn opt_parser
+      exit(false)
+    end
+
+    options[:toplevel] = ARGV[0]
+    options[:action] = ARGV[1]
+    options[:check_file] = ARGV[2]
+
+    options
+  end
+end
+
+AutoRollback.run