confctl 2.2.3 → 3.0.0

data/lib/confctl/nix.rb

@@ -1,450 +1,43 @@
-require 'confctl/utils/file'
-require 'digest'
-require 'etc'
-require 'json'
-require 'securerandom'
-require 'tempfile'
+require 'forwardable'
+require_relative 'config_type'
+require_relative 'nix_legacy'
+require_relative 'nix_flake'
 
 module ConfCtl
   class Nix
+    extend Forwardable
+
     # Create a new instance without access to {ConfCtl::Settings}, i.e. when
     # called outside of cluster configuration directory.
     # @return [Nix]
     def self.stateless(show_trace: false, max_jobs: 'auto')
-      new(show_trace:, max_jobs:)
+      new(show_trace: show_trace, max_jobs: max_jobs)
     end
 
-    include Utils::File
-
     def initialize(conf_dir: nil, show_trace: false, max_jobs: nil, cores: nil)
       @conf_dir = conf_dir || ConfDir.path
-      @show_trace = show_trace
-      @max_jobs = max_jobs || Settings.instance.max_jobs
-      @cores = cores
-      @cmd = SystemCommand.new
-    end
-
-    def confctl_settings
-      out_link = File.join(
-        cache_dir,
-        'build',
-        'settings.json'
-      )
-
-      with_cache(out_link) do
-        with_argument({
-          confDir: conf_dir,
-          build: :confctl
-        }) do |arg|
-          cmd_args = [
-            'nix-build',
-            '--arg', 'jsonArg', arg,
-            '--out-link', out_link,
-            (show_trace ? '--show-trace' : nil),
-            (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
-            (cores ? ['--cores', cores.to_s] : nil),
-            ConfCtl.nix_asset('evaluator.nix')
-          ].flatten.compact
-
-          cmd.run(*cmd_args)
-        end
-      end
-
-      demodulify(JSON.parse(File.read(out_link))['confctl'])
-    end
-
-    # Returns an array with options from all confctl modules
-    # @return [Array]
-    def module_options
-      options = nix_instantiate({
-        confDir: conf_dir,
-        build: :moduleOptions
-      })
-    end
-
-    # Returns an array with machine fqdns
-    # @return [Array<String>]
-    def list_machine_fqdns
-      nix_instantiate({
-        confDir: conf_dir,
-        build: :list
-      })['machines']
-    end
-
-    # Return machines and their config in a hash
-    # @return [Hash]
-    def list_machines
-      out_link = File.join(
-        cache_dir,
-        'build',
-        'machine-list.json'
-      )
-
-      with_cache(out_link) do
-        with_argument({
-          confDir: conf_dir,
-          build: :info
-        }, core_swpins: true) do |arg|
-          cmd_args = [
-            'nix-build',
-            '--arg', 'jsonArg', arg,
-            '--out-link', out_link,
-            (show_trace ? '--show-trace' : nil),
-            (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
-            (cores ? ['--cores', cores.to_s] : nil),
-            ConfCtl.nix_asset('evaluator.nix')
-          ].flatten.compact
-
-          cmd.run(*cmd_args)
-        end
-      end
-
-      demodulify(JSON.parse(File.read(out_link)))
-    end
-
-    def list_swpins_channels
-      nix_instantiate({
-        confDir: conf_dir,
-        build: :listSwpinsChannels
-      })
-    end
-
-    # Evaluate swpins for host
-    # @return [Hash]
-    def eval_core_swpins
-      with_argument({
-        confDir: conf_dir,
-        build: :evalCoreSwpins
-      }) do |arg|
-        out_link = File.join(
-          cache_dir,
-          'build',
-          'core.swpins'
-        )
-
-        cmd_args = [
-          'nix-build',
-          '--arg', 'jsonArg', arg,
-          '--out-link', out_link,
-          (show_trace ? '--show-trace' : nil),
-          (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
-          (cores ? ['--cores', cores.to_s] : nil),
-          ConfCtl.nix_asset('evaluator.nix')
-        ].flatten.compact
-
-        out, = cmd.run(*cmd_args).stdout
-
-        JSON.parse(File.read(out.strip))
-      end
-    end
-
-    # Evaluate swpins for hosts
-    # @param hosts [Array<String>]
-    # @return [Hash] host => swpins
-    def eval_host_swpins(hosts)
-      with_argument({
-        confDir: conf_dir,
-        build: :evalHostSwpins,
-        machines: hosts
-      }, core_swpins: true) do |arg|
-        out_link = File.join(
-          cache_dir,
-          'build',
-          "#{Digest::SHA256.hexdigest(hosts.join(','))[0..11]}.swpins"
-        )
-
-        cmd_args = [
-          'nix-build',
-          '--arg', 'jsonArg', arg,
-          '--out-link', out_link,
-          (show_trace ? '--show-trace' : nil),
-          (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
-          (cores ? ['--cores', cores.to_s] : nil),
-          ConfCtl.nix_asset('evaluator.nix')
-        ].flatten.compact
-
-        out, = cmd.run(*cmd_args)
-
-        JSON.parse(File.read(out.strip))
-      end
-    end
-
-    # Build config.system.build.toplevel for selected hosts
-    #
-    # @param hosts [Array<Machine>]
-    # @param swpin_paths [Hash]
-    # @param host_swpin_specs [Hash]
-    # @param time [Time]
-    #
-    # @yieldparam type [:build, :fetch]
-    # @yieldparam progress [Integer]
-    # @yieldparam total [Integer]
-    # @yieldparam path [String]
-    #
-    # @return [Hash<String, Generation::Build>]
-    def build_attributes(hosts: [], swpin_paths: {}, host_swpin_specs: {}, time: nil, &block)
-      with_argument({
-        confDir: conf_dir,
-        build: :toplevel,
-        machines: hosts
-      }) do |arg|
-        time ||= Time.now
-        ret_generations = {}
-        out_link = File.join(cache_dir, 'build', "#{SecureRandom.hex(4)}.build")
-
-        cmd_args = [
-          '--arg', 'jsonArg', arg,
-          '--out-link', out_link,
-          (show_trace ? '--show-trace' : nil),
-          (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
-          (cores ? ['--cores', cores.to_s] : nil),
-          ConfCtl.nix_asset('evaluator.nix')
-        ].flatten.compact
-
-        nb = NixBuild.new(cmd_args, swpin_paths)
-        nb.run(&block)
-
-        begin
-          host_results = JSON.parse(File.read(out_link))
-          host_results.each do |host, result|
-            host_generations = Generation::BuildList.new(host)
-            generation = host_generations.find(result['attribute'], swpin_paths)
-
-            if generation.nil?
-              generation = Generation::Build.new(host)
-              generation.create(
-                result['attribute'],
-                result['autoRollback'],
-                swpin_paths,
-                host_swpin_specs[host],
-                date: time
-              )
-              generation.save
-            end
-
-            host_generations.current = generation
-            ret_generations[host] = generation
-          end
-        ensure
-          unlink_if_exists(out_link)
-        end
-
-        ret_generations
-      end
-    end
-
-    # @param machine [Machine]
-    # @param paths [Array<String>]
-    #
-    # @yieldparam progress [Integer]
-    # @yieldparam total [Integer]
-    # @yieldparam path [String]
-    #
-    # @return [Boolean]
-    def copy(machine, paths, &)
-      if machine.localhost?
-        true
-      elsif machine.carried?
-        cp = NixCopy.new(machine.carrier_machine.target_host, paths)
-        cp.run!(&).success?
-      else
-        cp = NixCopy.new(machine.target_host, paths)
-        cp.run!(&).success?
-      end
-    end
-
-    # @param machine [Machine]
-    # @param generation [Generation::Build]
-    # @param action [String]
-    # @return [Boolean]
-    def activate(machine, generation, action)
-      args = [File.join(generation.toplevel, 'bin/switch-to-configuration'), action]
-
-      MachineControl.new(machine).execute!(*args).success?
-    end
-
-    # @param machine [Machine]
-    # @param generation [Generation::Build]
-    # @param action [String]
-    # @return [Boolean]
-    def activate_with_rollback(machine, generation, action)
-      check_file = File.join('/run', "confctl-confirm-#{SecureRandom.hex(3)}")
-      timeout = machine['autoRollback']['timeout']
-      logger = NullLogger.new
-
-      args = [
-        generation.auto_rollback,
-        '-t', timeout,
-        generation.toplevel,
-        action,
-        check_file
-      ]
-
-      activation_success = nil
-
-      activation_thread = Thread.new do
-        activation_success = MachineControl.new(machine).execute!(*args).success?
-      end
-
-      # Wait for the configuration to be switched
-      t = Time.now
-
-      loop do
-        out, = MachineControl.new(machine, logger:).execute!('cat', check_file, '2>/dev/null')
-        stripped = out.strip
-        break if stripped == 'switched' || ((t + timeout + 10) < Time.now && stripped != 'switching')
-
-        sleep(1)
-      end
-
-      # Confirm it
-      10.times do
-        break if MachineControl.new(machine, logger:).execute!('sh', '-c', "'echo confirmed > #{check_file}'").success?
-      end
-
-      activation_thread.join
-      activation_success
-    end
-
-    # @param machine [Machine]
-    # @param toplevel [String]
-    # @return [Boolean]
-    def set_profile(machine, toplevel)
-      args = [
-        'nix-env',
-        '-p', machine.profile,
-        '--set', toplevel
-      ]
-
-      MachineControl.new(machine).execute!(*args).success?
-    end
-
-    # @param machine [Machine]
-    # @param toplevel [String]
-    # @return [Boolean]
-    def set_carried_profile(machine, toplevel)
-      args = [
-        'carrier-env',
-        '-p', machine.profile,
-        '--set', toplevel
-      ]
-
-      MachineControl.new(machine.carrier_machine).execute!(*args).success?
-    end
-
-    # @param packages [Array<String>]
-    # @param command [String]
-    # @return [Boolean]
-    def run_command_in_shell(packages: [], command: nil)
-      args = ['nix-shell']
-
-      if packages.any?
-        args << '-p'
-        args.concat(packages)
-      end
-
-      args << '--command'
-      args << command
-
-      cmd.run!(*args, env: { 'shellHook' => nil }).success?
-    end
-
-    # @param machine [Machine]
-    # @yieldparam progress [NixCollectGarbage::Progress]
-    # @return [Boolean]
-    def collect_garbage(machine, &)
-      gc = NixCollectGarbage.new(machine)
-      gc.run!(&).success?
-    end
-
-    protected
-
-    attr_reader :conf_dir, :show_trace, :max_jobs, :cores, :cmd
-
-    # Execute block only if `out_link` does not exist or conf dir has changed
-    # @param out_link [String] out link path
-    def with_cache(out_link)
-      unchanged = false
-
-      if File.exist?(out_link)
-        unchanged = ConfDir.unchanged?
-
-        if unchanged
-          cache_mtime = ConfDir.state_mtime
-
-          if cache_mtime
-            begin
-              st = File.lstat(out_link)
-            rescue Errno::ENOENT
-              # pass
-            else
-              if st.mtime > cache_mtime
-                Logger.instance << "Using #{out_link}\n"
-                return
-              end
-            end
-          end
-        end
-      end
-
-      Logger.instance << "Building #{out_link}\n"
-
-      unless unchanged
-        Logger.instance << "Updating configuration cache\n"
-        ConfDir.update_state
-      end
-      yield
-    end
-
-    # @param hash [Hash]
-    # @param core_swpins [Boolean]
-    # @yieldparam file [String]
-    def with_argument(hash, core_swpins: false)
-      if core_swpins
-        paths = Swpins::Core.get.pre_evaluated_store_paths
-        hash[:coreSwpins] = paths if paths
-      end
-
-      f = Tempfile.new('confctl')
-      f.puts(hash.to_json)
-      f.close
-      yield(f.path)
-    ensure
-      f.unlink
-    end
-
-    def nix_instantiate(hash, **opts)
-      with_argument(hash, **opts) do |arg|
-        cmd_args = [
-          'nix-instantiate',
-          '--eval',
-          '--json',
-          '--strict',
-          '--read-write-mode',
-          '--arg', 'jsonArg', arg,
-          (show_trace ? '--show-trace' : nil),
-          (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
-          (cores ? ['--cores', cores.to_s] : nil),
-          ConfCtl.nix_asset('evaluator.nix')
-        ].flatten.compact
-
-        out, = cmd.run(*cmd_args).stdout
-
-        demodulify(JSON.parse(out))
-      end
-    end
-
-    def demodulify(value)
-      if value.is_a?(Array)
-        value.each { |item| demodulify(item) }
-      elsif value.is_a?(Hash)
-        value.delete('_module')
-        value.each_value { |v| demodulify(v) }
-      end
-    end
-
-    def cache_dir
-      ConfDir.cache_dir
-    end
+      impl_klass = ConfigType.flake?(@conf_dir) ? NixFlake : NixLegacy
+      @impl = impl_klass.new(conf_dir: @conf_dir, show_trace: show_trace, max_jobs: max_jobs, cores: cores)
+    end
+
+    def_delegators :@impl,
+                   :confctl_settings,
+                   :module_options,
+                   :list_machine_fqdns,
+                   :list_machines,
+                   :build_attributes,
+                   :copy,
+                   :activate,
+                   :activate_with_rollback,
+                   :set_profile,
+                   :set_carried_profile,
+                   :run_command_in_shell,
+                   :collect_garbage,
+                   :list_swpins_channels,
+                   :eval_core_swpins,
+                   :eval_host_swpins,
+                   :eval_inputs_info,
+                   :eval_inputs,
+                   :eval_json
   end
 end

data/lib/confctl/nix_build_flake.rb

@@ -0,0 +1,95 @@
+module ConfCtl
+  class NixBuildFlake
+    # @param args [Array<String>]
+    # @param chdir [String]
+    def initialize(args, chdir:)
+      @args = args
+      @chdir = chdir
+      @build_progress = 0
+      @build_total = 0
+      @fetch_progress = 0
+      @fetch_total = 0
+    end
+
+    # @yieldparam type [:build, :fetch]
+    # @yieldparam progress [Integer]
+    # @yieldparam total [Integer]
+    # @yieldparam path [String]
+    def run(&block)
+      cmd = SystemCommand.new
+
+      line_buf = StdLineBuffer.new do |_out, err|
+        parse_line(err, &block) if err && block
+      end
+
+      ret = cmd.run('nix', 'build', *args, chdir: chdir, &line_buf.feed_block)
+
+      line_buf.flush
+      ret
+    end
+
+    protected
+
+    attr_reader :args, :chdir
+
+    def parse_line(line)
+      # Beware that nix can fetch/build stuff even before those two
+      # summary lines are printed. Therefore we report progress with total=0
+      # (indeterminate) until the total becomes known.
+
+      # Nix >= around 2.11
+      case line
+      when /^this derivation will be built:/
+        @build_total = 1
+        return
+      when /^these (\d+) derivations will be built:/
+        @build_total = ::Regexp.last_match(1).to_i
+        return
+      when /^this path will be fetched /
+        @fetch_total = 1
+        return
+      when /^these (\d+) paths will be fetched /
+        @fetch_total = ::Regexp.last_match(1).to_i
+        return
+      end
+
+      # Nix < around 2.11, we can drop this in the future
+      if /^these derivations will be built:/ =~ line
+        @in_derivation_list = true
+        @in_fetch_list = false
+        return
+      elsif /^these paths will be fetched / =~ line
+        @in_derivation_list = false
+        @in_fetch_list = true
+        return
+      end
+
+      if @in_derivation_list
+        if %r{^\s+/nix/store/} =~ line
+          @build_total += 1
+          return
+        else
+          @in_derivation_list = false
+          @build_total += @build_progress
+        end
+
+      elsif @in_fetch_list
+        if %r{^\s+/nix/store/} =~ line
+          @fetch_total += 1
+          return
+        else
+          @in_fetch_list = false
+          @fetch_total += @fetch_progress
+        end
+      end
+
+      if /^building '([^']+)/ =~ line
+        @build_progress += 1
+        yield(:build, @build_progress, @build_total, ::Regexp.last_match(1))
+      elsif /^copying path '([^']+)/ =~ line
+        @fetch_progress += 1
+        yield(:fetch, @fetch_progress, @fetch_total, ::Regexp.last_match(1))
+      end
+    end
+  end
+end

data/lib/confctl/nix_copy.rb

@@ -1,10 +1,12 @@
 module ConfCtl
   class NixCopy
     # @param target [String]
+    # @param port [Integer]
     # @param store_paths [Array<String>]
-    def initialize(target, store_paths)
+    def initialize(target, store_paths, port: 22)
       @target = target
       @store_paths = store_paths
+      @port = port.to_i
       @total = nil
       @progress = 0
     end
@@ -23,6 +25,7 @@ module ConfCtl
         'nix-copy-closure',
         '--to', "root@#{target}",
         *store_paths,
+        env: nix_ssh_env,
         &line_buf.feed_block
       )
 
@@ -32,7 +35,16 @@ module ConfCtl
 
     protected
 
-    attr_reader :target, :store_paths
+    attr_reader :target, :store_paths, :port
+
+    def nix_ssh_env
+      return {} if port == 22
+
+      ssh_opts = [ENV.fetch('NIX_SSHOPTS', nil), "-p #{port}"].compact.join(' ').strip
+      {
+        'NIX_SSHOPTS' => ssh_opts
+      }
+    end
 
     def parse_line(line)
       if @total.nil? && /^copying (\d+) paths/ =~ line