foreman_opentofu 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aac98f7ab29438afb053493e3ef572e5374d5140eec49cc67e4f4d9787776a08
-  data.tar.gz: 85f7eb01a3b220b06b357c5ef9b0270bbf49fc19ea286f7e421964871f7bf726
+  metadata.gz: 8eceff0c5cce2e6fde716720f6262051d3ca5cd6355ca21db998d631f559eb78
+  data.tar.gz: 7e4ba716aca8ce9e6fede8abb74cfe33aae9838b02646a4804d008c9203f706a
 SHA512:
-  metadata.gz: 3a8231fb351a2f809d768880ed32dd0be78b13333453f4d5fbb71fbcc4c801dc963c0b08beeed69ba916c307c669856d17e8513d1cdb4923f25e0c6b49487e73
-  data.tar.gz: c221f8852cfedf5d07bb40bb93832c2512b77b59b58c043dd8f23cfdf61ee102321be84825158e61981bcd1255b4bc2b6c18fa589d368c5c0d2df16dc2697994
+  metadata.gz: 2db6f7efade936b283136c2b47dea0bd8cded850c94cdae3ef085f0a7f38c388f849f3ea7cb0fc398c4c3171c47cf0737d04b000c531373b8757cdd8607a3dc7
+  data.tar.gz: c0a8e35c4b2d39281bfbad14109be06b37f066b71738defbaff167bb235dee73978fa39811d5c07a25d258f23527dffae02e8ad4d9af186b3e8065b3ddee5edf

data/app/controllers/concerns/foreman_opentofu/compute_resources_controller.rb

@@ -0,0 +1,22 @@
+module ForemanOpentofu
+  module ComputeResourcesController
+    extend ActiveSupport::Concern
+
+    included do
+      prepend Overrides
+    end
+
+    module Overrides
+      def provider_selected
+        return super unless params[:opentofu_provider_selected]
+
+        @compute_resource = ComputeResource.new_provider(provider: 'Tofu')
+        @compute_resource.opentofu_provider = params[:provider]
+        default_template = @compute_resource.opentofu_template
+        @compute_resource.opentofu_template_id = default_template.id if default_template
+
+        render partial: 'form'
+      end
+    end
+  end
+end

data/app/lib/foreman_opentofu/concerns/base_template_scope_extensions.rb

@@ -64,22 +64,24 @@ module ForemanOpentofu
       end
       def vm_attributes(skip_list = [])
         available_attributes = @compute_resource.available_attributes
-        data = {}
-        res = ''
-        @cr_attrs.each do |key, value|
-          next if skip_list.include? key
+        skipped_attributes = skip_list.map(&:to_s)
+        data = @cr_attrs.each_with_object({}) do |(key, value), attributes|
+          next if skipped_attributes.include?(key.to_s)
 
-          conf = available_attributes[key]
-          if conf.nil?
+          config = available_attributes[key]
+          unless config
             Rails.logger.warn("Attribute #{key.inspect} is not supported.")
             next
           end
-          next if conf['group'] != 'vm'
-          next if value.blank? && !conf['mandatory']
+          next unless vm_attribute_included?(config, value)
 
-          data[key] = format_value(value, conf['type'])
+          attributes[key] = format_value(value, config['type'])
         end
-        res << to_hcl(data, snippet: true)
+        to_hcl(data, snippet: true)
+      end
+
+      def vm_attribute_included?(config, value)
+        config['group'] == 'vm' && (value.present? || config['mandatory'])
       end
 
       def backend_block
@@ -108,6 +110,8 @@ module ForemanOpentofu
       end
 
       def build_nics
+        return render_provider_data(@compute_resource.render_nic(nil, self, 0)) if @compute_resource.tofu_provider.nic_renderer_collection?
+
         nics_from_cr_attrs.each_with_index.map do |nic, index|
           data = @compute_resource.render_nic(nic, self, index)
           render_provider_data(data)

data/app/models/concerns/foreman_opentofu/compute_attribute_validation.rb

@@ -0,0 +1,25 @@
+module ForemanOpentofu
+  module ComputeAttributeValidation
+    extend ActiveSupport::Concern
+
+    included do
+      validate :validate_opentofu_vm_attributes
+    end
+
+    private
+
+    def validate_opentofu_vm_attributes
+      return unless compute_resource.is_a?(ForemanOpentofu::Tofu)
+
+      args = compute_resource.default_attributes.merge(vm_attrs.deep_dup).to_h.symbolize_keys
+      compute_resource.normalize_vm_args_collections!(args)
+      provider = compute_resource.tofu_provider
+
+      begin
+        provider.validate_vm!(args, compute_resource)
+      rescue ArgumentError => e
+        errors.add(:base, e.message)
+      end
+    end
+  end
+end

data/app/models/concerns/foreman_opentofu/power_commands.rb

@@ -0,0 +1,57 @@
+module ForemanOpentofu
+  module PowerCommands
+    def start_vm(name)
+      return change_vm_power(name, 'on') if tofu_provider.respond_to?(:power_state_attributes)
+
+      output = client({ 'name' => name, 'power_state' => 'on' }).run_create
+      output['vm']['power_state'] == 'on'
+    end
+
+    def stop_vm(name)
+      return change_vm_power(name, 'off') if tofu_provider.respond_to?(:power_state_attributes)
+
+      output = client({ 'name' => name, 'power_state' => 'off' }).run_create
+      output['vm']['power_state'] == 'off'
+    end
+
+    private
+
+    def change_vm_power(identifier, state)
+      vm_command_errors('change power') do
+        tf_state = power_state_record(identifier)
+        output = power_state_output(tf_state)
+        result = client(power_state_arguments(output, tf_state.name, state)).run_create(power_only: true)
+        verify_power_result(result, state)
+      end
+    end
+
+    def power_state_record(identifier)
+      tf_state = TfState.find_by(uuid: identifier) || TfState.find_by(name: identifier)
+      raise "No OpenTofu state found for #{identifier}" unless tf_state
+
+      tf_state
+    end
+
+    def power_state_output(tf_state)
+      output = client('name' => tf_state.name).run_output
+      valid = output['vm'].is_a?(Hash) && output['vm'].present?
+      raise 'Power control requires VM output from the provisioning template.' unless valid
+
+      output
+    end
+
+    def power_state_arguments(output, name, state)
+      arguments = ComputeVM.new(self, output).to_h
+      arguments.merge!(tofu_provider.power_state_attributes(state))
+      arguments['name'] = name
+      arguments
+    end
+
+    def verify_power_result(result, state)
+      vm = ComputeVM.new(self, result)
+      raise "OpenTofu did not return the requested power state #{state}" unless vm.power == state
+
+      vm
+    end
+  end
+end

data/app/models/concerns/foreman_opentofu/vm_command_collection_normalization.rb

@@ -1,7 +1,5 @@
 module ForemanOpentofu
   module VMCommandCollectionNormalization
-    private
-
     def normalize_vm_args_collections!(args)
       args[:image_id] = args[:image] if args.key?(:image)
       normalize_vm_boolean_args!(args)
@@ -13,6 +11,8 @@ module ForemanOpentofu
       end
     end
 
+    private
+
     def normalize_vm_boolean_args!(args)
       vm_boolean_keys.each do |key|
         next unless args.key?(key)

data/app/models/concerns/orchestration/tofu/compute.rb

@@ -6,6 +6,8 @@ module Orchestration
       def match_macs_to_nics(fog_attr)
         return super unless compute_resource.is_a?(ForemanOpentofu::Tofu)
 
+        return super if compute_resource.tofu_provider.respond_to?(:select_nic_for_mac)
+
         interfaces.select(&:physical?).each do |nic|
           mac = vm.send(fog_attr)
           logger.debug "Orchestration::Compute: nic #{nic.inspect} assigned to #{vm.inspect}"

data/app/models/foreman_opentofu/compute_vm.rb

@@ -21,6 +21,17 @@ module ForemanOpentofu
       self['status'] || self['power'] || self['power_state']
     end
 
+    def state
+      power
+    end
+
+    def reload
+      refreshed = @provider.find_vm_by_uuid(self['identity'])
+      @attributes = refreshed.to_h.deep_stringify_keys
+      define_dynamic_readers!
+      self
+    end
+
     # TODO: add definitions for different power on/off values
     def ready?
       power.to_s == 'on' || power.to_s == 'running'
@@ -76,6 +87,17 @@ module ForemanOpentofu
       instance_eval(&block)
     end
 
+    # Foreman's MAC orchestration expects NIC objects with attribute readers.
+    def interfaces
+      value = attribute_value('interfaces_attributes') || attribute_value('interfaces') || []
+      value = value.sort_by { |key, _| key.to_i }.map(&:last) if value.is_a?(Hash)
+      Array(value).map { |attrs| OpenStruct.new(attrs) }
+    end
+
+    def select_nic(nics, nic)
+      @provider.tofu_provider.select_nic_for_mac(nics, nic)
+    end
+
     def volumes_attributes
       vols_attrs = attribute_value('volumes_attributes')
       return vols_attrs if vols_attrs.is_a?(Hash)

data/app/models/foreman_opentofu/opentofu_vm_commands.rb

@@ -1,11 +1,14 @@
 module ForemanOpentofu
   module OpentofuVMCommands
     include ForemanOpentofu::VMCommandCollectionNormalization
+    include ForemanOpentofu::PowerCommands
 
     def find_vm_by_uuid(uuid)
       vm_command_errors('find vm') do
         tf_state = ForemanOpentofu::TfState.find_by(uuid: uuid)
         data = client({ 'name' => tf_state&.name }).run_output
+        # trying to add a volume with invalid parameter may persist in the volumes_attributes-hash
+        data['volumes_attributes'] = data['volumes_attributes']&.delete_if { |_idx, vol| vol.key?('id') && vol['id'].nil? }
         ComputeVM.new(self, data)
       end
     end
@@ -15,6 +18,8 @@ module ForemanOpentofu
         args = default_attributes.merge(args).to_h.symbolize_keys
         normalize_vm_args_collections!(args)
         args = prefill_mandatory_attributes(args).merge(args)
+        return ComputeVM.new(self, args.deep_stringify_keys) unless tofu_provider.plan_on_new_vm?
+
         executor = client(args)
         data = executor.run_new
         attrs = data['resource_changes'].first['change']['after'] || {}
@@ -24,11 +29,13 @@ module ForemanOpentofu
     end
 
     def create_vm(args = {})
+      args = default_attributes.merge(args).to_h.symbolize_keys
+      normalize_vm_args_collections!(args)
+      tofu_provider.validate_vm!(args, self)
+
       vm_command_errors('create vm') do
-        args = default_attributes.merge(args).to_h.symbolize_keys
-        normalize_vm_args_collections!(args)
         executor = client(args)
-        output = executor.run_create
+        output = executor.run_create(cleanup_on_failure: true)
         ComputeVM.new(self, output)
       end
     end
@@ -42,24 +49,16 @@ module ForemanOpentofu
       tf_state.destroy
     end
 
-    def start_vm(name)
-      output = client({ 'name' => name, 'power_state' => 'on' }).run_create
-      output['vm']['power_state'] == 'on'
-    end
-
-    def stop_vm(name)
-      output = client({ 'name' => name, 'power_state' => 'off' }).run_create
-      output['vm']['power_state'] == 'off'
-    end
-
     def save_vm(uuid, attrs)
       old_attrs = vm_compute_attributes_for(uuid).to_h.deep_stringify_keys
       tf_state = TfState.find_by(uuid: uuid)
       raise StandardError, "VM with UUID #{uuid} does not exist" unless tf_state
+      new_attrs = attrs.to_h.deep_stringify_keys
+      merged_attrs = old_attrs.merge(new_attrs).deep_symbolize_keys
+      normalize_vm_args_collections!(merged_attrs)
+      tofu_provider.validate_vm!(merged_attrs, self)
+
       vm_command_errors('update vm') do
-        new_attrs = attrs.to_h.deep_stringify_keys
-        merged_attrs = old_attrs.merge(new_attrs).deep_symbolize_keys
-        normalize_vm_args_collections!(merged_attrs)
         data = client({ 'name' => tf_state.name }.merge(merged_attrs)).run_create(raise_if_recreate: true)
         ComputeVM.new(self, data)
       end
@@ -85,13 +84,13 @@ module ForemanOpentofu
       yield
     rescue StandardError => e
       Foreman::Logging.exception("Caught #{provider} error", e)
-      raise ::Foreman::WrappedException.new(
-        e,
+      raise ::Foreman::Exception.new(
         N_(
-          "Foreman could not find a required %<provider>s resource in #{method_name}. " \
-          'Check if Foreman has the required permissions and the resource exists. Reason: %<error>s'
+          '%<provider>s failed to %<method>s. Reason: %<error>s'
         ),
-        { provider: provider, error: e.message }
+        provider: provider,
+        method: method_name,
+        error: e.message
       )
     end
 

data/app/models/foreman_opentofu/tofu.rb

@@ -21,9 +21,9 @@ module ForemanOpentofu
     include ComputeResourceCaching
     include KeyPairComputeResource
     validates :provider, presence: true, inclusion: { in: %w[Tofu] }
-    validates :url, presence: true
-    validates :user, presence: true
-    validates :password, presence: true
+    validates :url, presence: true, if: -> { mandatory_compute_resource_attribute?('url') }
+    validates :user, presence: true, if: -> { mandatory_compute_resource_attribute?('user') }
+    validates :password, presence: true, if: -> { mandatory_compute_resource_attribute?('password') }
 
     # alias_attribute :username, :user
     # alias_attribute :endpoint, :url
@@ -67,9 +67,11 @@ module ForemanOpentofu
     end
 
     def opentofu_template
-      return ProvisioningTemplate.find(attrs[:opentofu_template_id]) if attrs.key? :opentofu_template_id
-      # or default-template for this opentofu_provider
-      nil
+      return ProvisioningTemplate.find(attrs[:opentofu_template_id]) if attrs[:opentofu_template_id].present?
+
+      ProvisioningTemplate.unscoped.find_by!(
+        name: tofu_provider.default_template
+      )
     end
 
     def opentofu_template_id
@@ -148,6 +150,12 @@ module ForemanOpentofu
 
     private
 
+    def mandatory_compute_resource_attribute?(name)
+      tofu_provider&.connection_attrs&.any? do |attribute|
+        attribute['name'] == name && attribute['mandatory']
+      end
+    end
+
     def set_vm_volumes_attributes(vm, vm_attrs)
       volumes = if vm.respond_to?(:volumes_attributes)
                   vm.volumes_attributes.values

data/app/services/foreman_opentofu/app_wrapper.rb

@@ -7,7 +7,6 @@ module ForemanOpentofu
     #   - manage temp-work-dir; problem: no auto-remove after finished :-(
     #   - handle ENVVars if applicable
     #   - handle stderr and stdout separately
-    #   - use JSON-output for easier parsing
     #   - do we need locking or has the object be atomic
 
     def initialize(workdir, opts = {})
@@ -42,6 +41,7 @@ module ForemanOpentofu
     def default_params
       [
         '-no-color',
+        '-json',
       ]
     end
 
@@ -76,12 +76,12 @@ module ForemanOpentofu
     end
 
     def output(params = [])
-      JSON.parse(tofu_execute('output', ['-json'].concat(parse_params(params))))
+      tofu_execute('output', parse_params(params))
     end
 
     # TODO: find better name ;-)
     def show_plan(params = [])
-      JSON.parse(tofu_execute('show', ['-json', planfile].concat(parse_params(params))))
+      tofu_execute('show', [planfile].concat(parse_params(params)))
     end
 
     def planned?
@@ -98,12 +98,71 @@ module ForemanOpentofu
 
     private
 
+    def filter_sensitive_output(output)
+      filter_json_output(output.to_s)
+    rescue JSON::ParserError
+      output.to_s.each_line.map do |line|
+        filter_json_output(line)
+      rescue JSON::ParserError
+        line
+      end.join
+    end
+
+    def filter_json_output(output)
+      json = JSON.parse(output)
+      return output unless json.is_a?(Hash)
+
+      filtered = ActiveSupport::ParameterFilter.new(Rails.application.config.filter_parameters).filter(json)
+      return output if filtered == json
+
+      filtered.to_json + (output.end_with?("\n") ? "\n" : '')
+    end
+
+    def readable_output(output)
+      output.to_s.each_line.map do |line|
+        event = JSON.parse(line)
+        next line.chomp unless event.is_a?(Hash)
+
+        message = event.fetch('@message', line.chomp)
+        detail = event.dig('diagnostic', 'detail') if event['type'] == 'diagnostic' && event['diagnostic'].is_a?(Hash)
+        [message, detail].compact.reject(&:empty?).join("\n")
+      rescue JSON::ParserError
+        line.chomp
+      end.join("\n")
+    end
+
+    def error_summary(output)
+      errors = output.to_s.each_line.filter_map do |line|
+        diagnostic_summary(JSON.parse(line))
+      rescue JSON::ParserError
+        # stderr and startup failures may still contain plain text.
+        next
+      end
+      errors.empty? ? output : errors.join("\n\n")
+    end
+
+    def diagnostic_summary(event)
+      return unless event.is_a?(Hash) && event['type'] == 'diagnostic'
+
+      diagnostic = event['diagnostic']
+      return unless diagnostic.is_a?(Hash) && diagnostic['severity'] == 'error'
+      return if diagnostic['summary'].to_s.empty?
+
+      message = "Error: #{diagnostic['summary']}"
+      detail = diagnostic['detail']
+
+      detail.to_s.empty? ? message : "#{message}\n#{detail}"
+    end
+
     def parse_params(params)
       params.is_a?(String) ? [params] : params
     end
 
     def tofu_execute(action, params = [], &block)
-      execute [base_command, action].concat(default_params).concat(params), &block
+      output = execute [base_command, action].concat(default_params).concat(params), &block
+      return JSON.parse(output) if %w[output show].include?(action) && !block
+
+      output
     end
 
     def command(cmd)
@@ -141,13 +200,19 @@ module ForemanOpentofu
       end
       ret = $CHILD_STATUS
       Rails.logger.info "#{cmd} returned #{ret.inspect}"
-      Rails.logger.debug output.to_s
+      filtered_output = filter_sensitive_output(output)
+      Rails.logger.debug filtered_output
+
+      readable = readable_output(filtered_output)
+
       unless ret.success?
-        Rails.logger.error "Command failed with output: #{output}"
+        Rails.logger.error "Command failed with output: #{readable}"
         # TODO: do we need to use a specific exception-type here?
-        raise "command failed with code #{ret.exitstatus}:\n#{output}"
+        raise error_summary(output)
       end
 
+      Rails.logger.info readable unless readable.empty?
+
       output
     end
   end

data/app/services/foreman_opentofu/opentofu_executer.rb

@@ -3,6 +3,11 @@ module ForemanOpentofu
     # 'destroy' works with TfState rather than the actual tf-file
     # also it might not receive all the necessary data to create a valid tf-file
     DRY_RUN_MODES = %w[destroy test].freeze
+    PROVIDER_VARIABLE_NAMES = {
+      'url' => :endpoint,
+      'user' => :username,
+      'password' => :password,
+    }.freeze
 
     def initialize(compute_resource, args = {})
       @compute_resource = compute_resource
@@ -21,11 +26,7 @@ module ForemanOpentofu
             f.write(@cr_attrs['user_data'])
           end
         end
-        tofu = AppWrapper.new(dir, variables: {
-          username: @compute_resource.user,
-          password: @compute_resource.password,
-          endpoint: @compute_resource.url,
-        })
+        tofu = AppWrapper.new(dir, variables: provider_variables)
         @use_backend = %w[create destroy output keygen].include?(mode)
         @token = create_token(@host_name) if @use_backend
         tofu.main_configuration = render_template(mode)
@@ -73,21 +74,11 @@ module ForemanOpentofu
       run('test', &:plan)
     end
 
-    def run_create(raise_if_recreate: false)
+    def run_create(raise_if_recreate: false, cleanup_on_failure: false, power_only: false)
       run('create') do |tofu|
-        if raise_if_recreate
-          # check the plan in advance to verify we do not replace the VM
-          tofu.plan
-          raise 'OpenTofu planned to re-create a resource; action aborted (check logs for details)!' if plan_wants_recreate? tofu.show_plan
-        end
-        begin
-          tofu.apply
-          attrs = tofu.output('vm_attrs')
-          ForemanOpentofu::TfState.find_by(name: @cr_attrs['name'])&.update(uuid: attrs['identity'])
-          attrs
-        rescue StandardError => e
-          handle_failed_create(tofu, e)
-        end
+        validate_power_plan(tofu) if power_only
+        validate_update_plan(tofu) if raise_if_recreate
+        apply_create(tofu, cleanup_on_failure)
       end
     end
 
@@ -122,6 +113,42 @@ module ForemanOpentofu
 
     private
 
+    def validate_power_plan(tofu)
+      tofu.plan
+      changes = tofu.show_plan.fetch('resource_changes', [])
+      return if changes.all? { |change| @compute_resource.tofu_provider.power_change_allowed?(change) }
+
+      raise 'OpenTofu planned changes other than server power; action aborted.'
+    end
+
+    def validate_update_plan(tofu)
+      tofu.plan
+      return unless plan_wants_recreate?(tofu.show_plan)
+
+      raise 'OpenTofu planned to re-create a resource; action aborted (check logs for details)!'
+    end
+
+    def apply_create(tofu, cleanup_on_failure)
+      tofu.apply
+      attrs = tofu.output('vm_attrs')
+      ForemanOpentofu::TfState.find_by(name: @cr_attrs['name'])&.update(uuid: attrs['identity'])
+      attrs
+    rescue StandardError => e
+      raise e unless cleanup_on_failure
+
+      handle_failed_create(tofu, e)
+    end
+
+    def provider_variables
+      @compute_resource.tofu_provider.connection_attrs.each_with_object({}) do |attribute, variables|
+        attribute_name = attribute['name'].to_s
+        variable_name = PROVIDER_VARIABLE_NAMES[attribute_name]
+        next unless variable_name
+
+        variables[variable_name] = @compute_resource.public_send(attribute_name)
+      end
+    end
+
     def render_template(mode)
       template = provision_template
       variables = {
@@ -160,7 +187,10 @@ module ForemanOpentofu
     end
 
     def plan_wants_recreate?(plan)
-      need_recreate = plan['resource_changes'].select { |res| (res.dig('change', 'actions') || []).include?('delete') }
+      need_recreate = plan['resource_changes'].select do |res|
+        actions = res.dig('change', 'actions') || []
+        actions.include?('delete') && actions.include?('create')
+      end
       need_recreate = @compute_resource.tofu_provider.filter_resource_changes need_recreate
       # {
       #   "address": "hcloud_server.node1",