confctl 1.0.0

data/lib/confctl/module_options.rb

@@ -0,0 +1,95 @@
+require 'cgi'
+
+module ConfCtl
+  class ModuleOptions
+    class Option
+      attr_reader :name, :description, :type, :default, :example, :declarations
+
+      def initialize(nixos_opt)
+        @name = nixos_opt['name']
+        @description = nixos_opt['description'] || 'This option has no description.'
+        @type = nixos_opt['type']
+        @default = extract_expression(nixos_opt['default'])
+        @example = extract_expression(nixos_opt['example'])
+        @declarations = nixos_opt['declarations'].map do |v|
+          raise "unable to place module '#{v}'" unless %r{/confctl/([^$]+)} =~ v
+
+          "<confctl/#{::Regexp.last_match(1)}>"
+        end
+      end
+
+      def md_description
+        tagless = description
+                  .gsub(%r{<literal>([^<]+)</literal>}, '`\1`')
+                  .gsub(%r{<option>([^<]+)</option>}, '`\1`')
+
+        CGI.unescapeHTML(tagless)
+      end
+
+      def nix_default
+        nixify(default)
+      end
+
+      def nix_example
+        example && nixify(example)
+      end
+
+      protected
+
+      def extract_expression(v)
+        if v.is_a?(Hash)
+          case v['_type']
+          when 'literalExpression'
+            NixLiteralExpression.new(v['text'])
+          else
+            raise "Unsupported expression type #{v['_type'].inspect}"
+          end
+        else
+          v
+        end
+      end
+
+      def nixify(v)
+        ConfCtl::NixFormat.to_nix(v)
+      end
+    end
+
+    # @return [Array<ModuleOptions::Option>]
+    attr_reader :options
+
+    # @param nix [Nix, nil]
+    def initialize(nix: nil)
+      @nix = nix || Nix.new
+      @options = []
+    end
+
+    def read
+      @options = nix.module_options.map do |opt|
+        Option.new(opt)
+      end
+    end
+
+    def confctl_settings
+      options.select do |opt|
+        opt.name.start_with?('confctl.') \
+          && !opt.name.start_with?('confctl.swpins.')
+      end
+    end
+
+    def swpin_settings
+      options.select { |opt| opt.name.start_with?('confctl.swpins.') }
+    end
+
+    def machine_settings
+      options.select { |opt| opt.name.start_with?('cluster.') }
+    end
+
+    def service_settings
+      options.select { |opt| opt.name.start_with?('services.') }
+    end
+
+    protected
+
+    attr_reader :nix
+  end
+end

data/lib/confctl/nix.rb

@@ -0,0 +1,382 @@
+require 'confctl/utils/file'
+require 'etc'
+require 'json'
+require 'securerandom'
+require 'tempfile'
+
+module ConfCtl
+  class Nix
+    # Create a new instance without access to {ConfCtl::Settings}, i.e. when
+    # called outside of cluster configuration directory.
+    # @return [Nix]
+    def self.stateless(show_trace: false, max_jobs: 'auto')
+      new(show_trace:, max_jobs:)
+    end
+
+    include Utils::File
+
+    def initialize(conf_dir: nil, show_trace: false, max_jobs: nil)
+      @conf_dir = conf_dir || ConfDir.path
+      @show_trace = show_trace
+      @max_jobs = max_jobs || Settings.instance.max_jobs
+      @cmd = SystemCommand.new
+    end
+
+    def confctl_settings
+      out_link = File.join(
+        cache_dir,
+        'build',
+        'settings.json'
+      )
+
+      with_cache(out_link) do
+        with_argument({
+          confDir: conf_dir,
+          build: :confctl
+        }) do |arg|
+          cmd_args = [
+            'nix-build',
+            '--arg', 'jsonArg', arg,
+            '--out-link', out_link,
+            (show_trace ? '--show-trace' : nil),
+            (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
+            ConfCtl.nix_asset('evaluator.nix')
+          ].flatten.compact
+
+          cmd.run(*cmd_args)
+        end
+      end
+
+      demodulify(JSON.parse(File.read(out_link))['confctl'])
+    end
+
+    # Returns an array with options from all confctl modules
+    # @return [Array]
+    def module_options
+      options = nix_instantiate({
+        confDir: conf_dir,
+        build: :moduleOptions
+      })
+    end
+
+    # Returns an array with machine fqdns
+    # @return [Array<String>]
+    def list_machine_fqdns
+      nix_instantiate({
+        confDir: conf_dir,
+        build: :list
+      })['machines']
+    end
+
+    # Return machines and their config in a hash
+    # @return [Hash]
+    def list_machines
+      out_link = File.join(
+        cache_dir,
+        'build',
+        'machine-list.json'
+      )
+
+      with_cache(out_link) do
+        with_argument({
+          confDir: conf_dir,
+          build: :info
+        }, core_swpins: true) do |arg|
+          cmd_args = [
+            'nix-build',
+            '--arg', 'jsonArg', arg,
+            '--out-link', out_link,
+            (show_trace ? '--show-trace' : nil),
+            (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
+            ConfCtl.nix_asset('evaluator.nix')
+          ].flatten.compact
+
+          cmd.run(*cmd_args)
+        end
+      end
+
+      demodulify(JSON.parse(File.read(out_link)))
+    end
+
+    def list_swpins_channels
+      nix_instantiate({
+        confDir: conf_dir,
+        build: :listSwpinsChannels
+      })
+    end
+
+    # Evaluate swpins for host
+    # @return [Hash]
+    def eval_core_swpins
+      with_argument({
+        confDir: conf_dir,
+        build: :evalCoreSwpins
+      }) do |arg|
+        out_link = File.join(
+          cache_dir,
+          'build',
+          'core.swpins'
+        )
+
+        cmd_args = [
+          'nix-build',
+          '--arg', 'jsonArg', arg,
+          '--out-link', out_link,
+          (show_trace ? '--show-trace' : nil),
+          (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
+          ConfCtl.nix_asset('evaluator.nix')
+        ].flatten.compact
+
+        out, = cmd.run(*cmd_args).stdout
+
+        JSON.parse(File.read(out.strip))
+      end
+    end
+
+    # Evaluate swpins for host
+    # @param host [String]
+    # @return [Hash]
+    def eval_host_swpins(host)
+      with_argument({
+        confDir: conf_dir,
+        build: :evalHostSwpins,
+        machines: [host]
+      }, core_swpins: true) do |arg|
+        out_link = File.join(
+          cache_dir,
+          'build',
+          "#{ConfCtl.safe_host_name(host)}.swpins"
+        )
+
+        cmd_args = [
+          'nix-build',
+          '--arg', 'jsonArg', arg,
+          '--out-link', out_link,
+          (show_trace ? '--show-trace' : nil),
+          (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
+          ConfCtl.nix_asset('evaluator.nix')
+        ].flatten.compact
+
+        out, = cmd.run(*cmd_args)
+
+        JSON.parse(File.read(out.strip))[host]
+      end
+    end
+
+    # Build config.system.build.toplevel for selected hosts
+    #
+    # @param hosts [Array<String>]
+    # @param swpin_paths [Hash]
+    # @param host_swpin_specs [Hash]
+    # @param time [Time]
+    #
+    # @yieldparam type [:build, :fetch]
+    # @yieldparam progress [Integer]
+    # @yieldparam total [Integer]
+    # @yieldparam path [String]
+    #
+    # @return [Hash<String, Generation::Build>]
+    def build_toplevels(hosts: [], swpin_paths: {}, host_swpin_specs: {}, time: nil, &block)
+      with_argument({
+        confDir: conf_dir,
+        build: :toplevel,
+        machines: hosts
+      }) do |arg|
+        time ||= Time.now
+        ret_generations = {}
+        out_link = File.join(cache_dir, 'build', "#{SecureRandom.hex(4)}.build")
+
+        cmd_args = [
+          '--arg', 'jsonArg', arg,
+          '--out-link', out_link,
+          (show_trace ? '--show-trace' : nil),
+          (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
+          ConfCtl.nix_asset('evaluator.nix')
+        ].flatten.compact
+
+        nb = NixBuild.new(cmd_args, swpin_paths)
+        nb.run(&block)
+
+        begin
+          host_toplevels = JSON.parse(File.read(out_link))
+          host_toplevels.each do |host, toplevel|
+            host_generations = Generation::BuildList.new(host)
+            generation = host_generations.find(toplevel, swpin_paths)
+
+            if generation.nil?
+              generation = Generation::Build.new(host)
+              generation.create(
+                toplevel,
+                swpin_paths,
+                host_swpin_specs[host],
+                date: time
+              )
+              generation.save
+            end
+
+            host_generations.current = generation
+            ret_generations[host] = generation
+          end
+        ensure
+          unlink_if_exists(out_link)
+        end
+
+        ret_generations
+      end
+    end
+
+    # @param machine [Machine]
+    # @param toplevel [String]
+    #
+    # @yieldparam progress [Integer]
+    # @yieldparam total [Integer]
+    # @yieldparam path [String]
+    #
+    # @return [Boolean]
+    def copy(machine, toplevel, &)
+      if machine.localhost?
+        true
+      else
+        cp = NixCopy.new(machine.target_host, toplevel)
+        cp.run!(&).success?
+      end
+    end
+
+    # @param machine [Machine]
+    # @param toplevel [String]
+    # @param action [String]
+    # @return [Boolean]
+    def activate(machine, toplevel, action)
+      args = [File.join(toplevel, 'bin/switch-to-configuration'), action]
+
+      MachineControl.new(machine).execute!(*args).success?
+    end
+
+    # @param machine [Machine]
+    # @param toplevel [String]
+    # @return [Boolean]
+    def set_profile(machine, toplevel)
+      args = [
+        'nix-env',
+        '-p', '/nix/var/nix/profiles/system',
+        '--set', toplevel
+      ]
+
+      MachineControl.new(machine).execute!(*args).success?
+    end
+
+    # @param packages [Array<String>]
+    # @param command [String]
+    # @return [Boolean]
+    def run_command_in_shell(packages: [], command: nil)
+      args = ['nix-shell']
+
+      if packages.any?
+        args << '-p'
+        args.concat(packages)
+      end
+
+      args << '--command'
+      args << command
+
+      cmd.run!(*args, env: { 'shellHook' => nil }).success?
+    end
+
+    # @param machine [Machine]
+    # @yieldparam progress [NixCollectGarbage::Progress]
+    # @return [Boolean]
+    def collect_garbage(machine, &)
+      gc = NixCollectGarbage.new(machine)
+      gc.run!(&).success?
+    end
+
+    protected
+
+    attr_reader :conf_dir, :show_trace, :max_jobs, :cmd
+
+    # Execute block only if `out_link` does not exist or conf dir has changed
+    # @param out_link [String] out link path
+    def with_cache(out_link)
+      unchanged = false
+
+      if File.exist?(out_link)
+        unchanged = ConfDir.unchanged?
+
+        if unchanged
+          cache_mtime = ConfDir.state_mtime
+
+          if cache_mtime
+            begin
+              st = File.lstat(out_link)
+            rescue Errno::ENOENT
+              # pass
+            else
+              if st.mtime > cache_mtime
+                Logger.instance << "Using #{out_link}\n"
+                return
+              end
+            end
+          end
+        end
+      end
+
+      Logger.instance << "Building #{out_link}\n"
+
+      unless unchanged
+        Logger.instance << "Updating configuration cache\n"
+        ConfDir.update_state
+      end
+      yield
+    end
+
+    # @param hash [Hash]
+    # @param core_swpins [Boolean]
+    # @yieldparam file [String]
+    def with_argument(hash, core_swpins: false)
+      if core_swpins
+        paths = Swpins::Core.get.pre_evaluated_store_paths
+        hash[:coreSwpins] = paths if paths
+      end
+
+      f = Tempfile.new('confctl')
+      f.puts(hash.to_json)
+      f.close
+      yield(f.path)
+    ensure
+      f.unlink
+    end
+
+    def nix_instantiate(hash, **opts)
+      with_argument(hash, **opts) do |arg|
+        cmd_args = [
+          'nix-instantiate',
+          '--eval',
+          '--json',
+          '--strict',
+          '--read-write-mode',
+          '--arg', 'jsonArg', arg,
+          (show_trace ? '--show-trace' : nil),
+          (max_jobs ? ['--max-jobs', max_jobs.to_s] : nil),
+          ConfCtl.nix_asset('evaluator.nix')
+        ].flatten.compact
+
+        out, = cmd.run(*cmd_args).stdout
+
+        demodulify(JSON.parse(out))
+      end
+    end
+
+    def demodulify(value)
+      if value.is_a?(Array)
+        value.each { |item| demodulify(item) }
+      elsif value.is_a?(Hash)
+        value.delete('_module')
+        value.each_value { |v| demodulify(v) }
+      end
+    end
+
+    def cache_dir
+      ConfDir.cache_dir
+    end
+  end
+end

data/lib/confctl/nix_build.rb

@@ -0,0 +1,108 @@
+module ConfCtl
+  class NixBuild
+    # @param args [Array<String>]
+    # @param swpin_paths [Hash<String>]
+    def initialize(args, swpin_paths)
+      @args = args
+      @swpin_paths = swpin_paths
+      @build_progress = 0
+      @build_total = 0
+      @fetch_progress = 0
+      @fetch_total = 0
+    end
+
+    # @yieldparam type [:build, :fetch]
+    # @yieldparam progress [Integer]
+    # @yieldparam total [Integer]
+    # @yieldparam path [String]
+    def run(&block)
+      cmd = SystemCommand.new
+
+      line_buf = StdLineBuffer.new do |_out, err|
+        parse_line(err, &block) if err && block
+      end
+
+      ret = cmd.run(
+        'nix-build',
+        *args,
+        env: { 'NIX_PATH' => build_nix_path(swpin_paths) },
+        &line_buf.feed_block
+      )
+
+      line_buf.flush
+      ret
+    end
+
+    protected
+
+    attr_reader :args, :swpin_paths
+
+    def parse_line(line)
+      # Beware that nix-build can fetch/build stuff even before those two
+      # summary lines are printed. Therefore we report progress with total=0
+      # (indeterminate) until the total becomes known.
+
+      # Nix >= around 2.11
+      case line
+      when /^this derivation will be built:/
+        @build_total = 1
+        return
+      when /^these (\d+) derivations will be built:/
+        @build_total = ::Regexp.last_match(1).to_i
+        return
+      when /^this path will be fetched /
+        @fetch_total = 1
+        return
+      when /^these (\d+) paths will be fetched /
+        @fetch_total = ::Regexp.last_match(1).to_i
+        return
+      end
+
+      # Nix < around 2.11, we can drop this in the future
+      if /^these derivations will be built:/ =~ line
+        @in_derivation_list = true
+        @in_fetch_list = false
+        return
+      elsif /^these paths will be fetched / =~ line
+        @in_derivation_list = false
+        @in_fetch_list = true
+        return
+      end
+
+      if @in_derivation_list
+        if %r{^\s+/nix/store/} =~ line
+          @build_total += 1
+          return
+        else
+          @in_derivation_list = false
+          @build_total += @build_progress
+        end
+
+      elsif @in_fetch_list
+        if %r{^\s+/nix/store/} =~ line
+          @fetch_total += 1
+          return
+        else
+          @in_fetch_list = false
+          @fetch_total += @fetch_progress
+        end
+      end
+
+      if /^building '([^']+)/ =~ line
+        @build_progress += 1
+        yield(:build, @build_progress, @build_total, ::Regexp.last_match(1))
+      elsif /^copying path '([^']+)/ =~ line
+        @fetch_progress += 1
+        yield(:fetch, @fetch_progress, @fetch_total, ::Regexp.last_match(1))
+      end
+    end
+
+    def build_nix_path(swpins)
+      paths = []
+      paths << "confctl=#{ConfCtl.root}"
+      paths.concat(swpins.map { |k, v| "#{k}=#{v}" })
+      paths.concat(Settings.instance.nix_paths)
+      paths.join(':')
+    end
+  end
+end

data/lib/confctl/nix_collect_garbage.rb

@@ -0,0 +1,64 @@
+module ConfCtl
+  # Run nix-collect-garbage on {Machine}
+  class NixCollectGarbage
+    class Progress
+      def initialize(line)
+        @line = line
+        @path = false
+        parse
+      end
+
+      def path?
+        !@path.nil?
+      end
+
+      # @return [String, nil]
+      attr_reader :path
+
+      def to_s
+        @line
+      end
+
+      protected
+
+      def parse
+        return unless %r{^deleting '(/nix/store/[^']+)'$} =~ @line
+
+        @path = ::Regexp.last_match(1)
+      end
+    end
+
+    # @param machine [String]
+    def initialize(machine)
+      @machine = machine
+    end
+
+    # @yieldparam progress [Progress]
+    # @return [TTY::Command::Result]
+    def run!
+      mc = MachineControl.new(machine)
+
+      line_buf = StdLineBuffer.new do |out, err|
+        next unless block_given?
+
+        # %d store paths deleted, %f MiB freed
+        yield(Progress.new(out)) if out
+
+        # finding garbage collector roots...
+        # removing stale link from '...'
+        # deleting garbage...
+        # deleting '/nix/store/...'
+        # ...
+        yield(Progress.new(err)) if err
+      end
+
+      ret = mc.execute!('nix-collect-garbage', &line_buf.feed_block)
+      line_buf.flush
+      ret
+    end
+
+    protected
+
+    attr_reader :machine
+  end
+end

data/lib/confctl/nix_copy.rb

@@ -0,0 +1,49 @@
+module ConfCtl
+  class NixCopy
+    # @param target [String]
+    # @param store_path [String]
+    def initialize(target, store_path)
+      @target = target
+      @store_path = store_path
+      @total = nil
+      @progress = 0
+    end
+
+    # @yieldparam progress [Integer]
+    # @yieldparam total [Integer]
+    # @yieldparam path [String]
+    def run!(&block)
+      cmd = SystemCommand.new
+
+      line_buf = StdLineBuffer.new do |_out, err|
+        parse_line(err, &block) if err && block
+      end
+
+      ret = cmd.run!(
+        'nix-copy-closure',
+        '--to', "root@#{target}",
+        store_path,
+        &line_buf.feed_block
+      )
+
+      line_buf.flush
+      ret
+    end
+
+    protected
+
+    attr_reader :target, :store_path
+
+    def parse_line(line)
+      if @total.nil? && /^copying (\d+) paths/ =~ line
+        @total = ::Regexp.last_match(1).to_i
+        return
+      end
+
+      return unless /^copying path '([^']+)/ =~ line
+
+      @progress += 1
+      yield(@progress, @total, ::Regexp.last_match(1))
+    end
+  end
+end