foreman_opentofu 0.0.5 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13aa21537aa49784d6b945e5e0169c20509c46e361e951b6db79a17d330559ef
-  data.tar.gz: e0f23b3286a236bd161421d8464518a9b47878d61e7ebd03ab41b361cb0b513f
+  metadata.gz: 8eceff0c5cce2e6fde716720f6262051d3ca5cd6355ca21db998d631f559eb78
+  data.tar.gz: 7e4ba716aca8ce9e6fede8abb74cfe33aae9838b02646a4804d008c9203f706a
 SHA512:
-  metadata.gz: f1459d8aa7ad4e99b3d43b2725093f7cdc06d7473073432ca8386f46b1d2db4a5ac3acad3ec7c3580830e2d13a11e25354713c9a702a1f3ac669a527a504137f
-  data.tar.gz: 4724e84ebc240ef60bdd95c65710fc3156f718628d38ba65570d8305df762be8883e00df13783002927589cdb2e7eed0304b98b414e5ac4e1edf9a48c5ca3d8c
+  metadata.gz: 2db6f7efade936b283136c2b47dea0bd8cded850c94cdae3ef085f0a7f38c388f849f3ea7cb0fc398c4c3171c47cf0737d04b000c531373b8757cdd8607a3dc7
+  data.tar.gz: c0a8e35c4b2d39281bfbad14109be06b37f066b71738defbaff167bb235dee73978fa39811d5c07a25d258f23527dffae02e8ad4d9af186b3e8065b3ddee5edf

data/README.md

@@ -337,6 +337,23 @@ bundle exec foreman start
 
 See details in [foreman plugin development](https://projects.theforeman.org/projects/foreman/wiki/How_to_Create_a_Plugin)
 
+
+### Testing
+
+#### Template Snapshots
+
+The OpenTofu templates are tested against recorded snapshots similar to the Provisioning Templates in Foreman.
+The snapshots are generated by rendering the templates with predefined fixture-data using the following command in the foreman-directory:
+
+```shell
+bundle exec rake foreman_opentofu:snapshots:generate RAILS_ENV=test
+```
+
+At the moment the fixture-data is defined in `app/services/foreman_opentofu/template_snapshot_service.rb`.
+
+Templates to test must be added to `test/fixtures/renderer/snapshots.yaml`.
+
+
 ## Contributing
 
 Fork and send a Pull Request or create Issue. Thank you.

data/app/controllers/api/v2/tf_states_controller.rb

@@ -2,10 +2,8 @@ module Api
   module V2
     class TfStatesController < ::Api::V2::BaseController
       include ::Api::Version2
+      include ::Foreman::Controller::FilterParameters
 
-      # TODO: verify this
-      # We don't require any of these methods for provisioning
-      # skip_before_action :require_login, :check_user_enabled, :session_expiry, :update_activity_time, :set_taxonomy, :authorize, unless: -> { preview? }
       skip_before_action :set_taxonomy
 
       # Allow HTTP POST methods without CSRF
@@ -14,6 +12,11 @@ module Api
       # overwrite authorize with the local token-based authorization
       before_action :authorize, except: [:destroy]
 
+      # Don't log tfstate response in logs as it can have sensitive information
+      skip_after_action :log_response_body
+
+      filter_parameters :version, :terraform_version, :serial, :lineage, :outputs, :resources, :tf_state, :check_results
+
       resource_description do
         api_version 'v2'
         api_base_url '/foreman_opentofu/api'
@@ -22,15 +25,13 @@ module Api
       def show
         state = ForemanOpentofu::TfState.find_by(name: params[:name])
 
-        if state
-          render plain: state.state, content_type: 'application/json'
-        else
-          render plain: '', status: :not_found
-        end
+        return head :not_found unless state
+
+        render plain: state.state, content_type: 'application/json'
       end
 
       def create
-        raw_state = request.body.read
+        raw_state = request.raw_post
         if raw_state.blank?
           render plain: 'Missing state body', status: :unprocessable_entity
           return
@@ -41,10 +42,9 @@ module Api
           state = ForemanOpentofu::TfState.find_or_initialize_by(name: params[:name])
           state.state = raw_state
           state.save!
-          render plain: '', status: :ok
-        rescue JSON::ParserError => e
-          Rails.logger.error("Invalid state JSON: #{e.message}")
-          render plain: 'Invalid state format', status: :unprocessable_entity
+          head :ok
+        rescue JSON::ParserError
+          render plain: 'Invalid state format', status: :bad_request
         end
       end
 

data/app/controllers/concerns/foreman_opentofu/compute_resources_controller.rb

@@ -0,0 +1,22 @@
+module ForemanOpentofu
+  module ComputeResourcesController
+    extend ActiveSupport::Concern
+
+    included do
+      prepend Overrides
+    end
+
+    module Overrides
+      def provider_selected
+        return super unless params[:opentofu_provider_selected]
+
+        @compute_resource = ComputeResource.new_provider(provider: 'Tofu')
+        @compute_resource.opentofu_provider = params[:provider]
+        default_template = @compute_resource.opentofu_template
+        @compute_resource.opentofu_template_id = default_template.id if default_template
+
+        render partial: 'form'
+      end
+    end
+  end
+end

data/app/lib/foreman_opentofu/concerns/base_template_scope_extensions.rb

@@ -64,22 +64,24 @@ module ForemanOpentofu
       end
       def vm_attributes(skip_list = [])
         available_attributes = @compute_resource.available_attributes
-        data = {}
-        res = ''
-        @cr_attrs.each do |key, value|
-          next if skip_list.include? key
+        skipped_attributes = skip_list.map(&:to_s)
+        data = @cr_attrs.each_with_object({}) do |(key, value), attributes|
+          next if skipped_attributes.include?(key.to_s)
 
-          conf = available_attributes[key]
-          if conf.nil?
+          config = available_attributes[key]
+          unless config
             Rails.logger.warn("Attribute #{key.inspect} is not supported.")
             next
           end
-          next if conf['group'] != 'vm'
-          next if value.blank? && !conf['mandatory']
+          next unless vm_attribute_included?(config, value)
 
-          data[key] = format_value(value, conf['type'])
+          attributes[key] = format_value(value, config['type'])
         end
-        res << to_hcl(data, snippet: true)
+        to_hcl(data, snippet: true)
+      end
+
+      def vm_attribute_included?(config, value)
+        config['group'] == 'vm' && (value.present? || config['mandatory'])
       end
 
       def backend_block
@@ -97,6 +99,8 @@ module ForemanOpentofu
       end
 
       def build_disks
+        return rendered_disk(0, disk_render_source.first).to_s if disk_renderer_collection?
+
         disk_render_source.each_with_index.filter_map do |disk, index|
           # drop removed disks; tofu will drop them automatically, if they are no longer defined
           next if disk.respond_to?(:[]) && disk['_delete'].to_i == 1
@@ -106,6 +110,8 @@ module ForemanOpentofu
       end
 
       def build_nics
+        return render_provider_data(@compute_resource.render_nic(nil, self, 0)) if @compute_resource.tofu_provider.nic_renderer_collection?
+
         nics_from_cr_attrs.each_with_index.map do |nic, index|
           data = @compute_resource.render_nic(nic, self, index)
           render_provider_data(data)
@@ -151,6 +157,15 @@ module ForemanOpentofu
           to_hcl(data, snippet: true)
         end
       end
+
+      def disk_renderer_collection?
+        return false unless @compute_resource.respond_to?(:tofu_provider)
+
+        provider = @compute_resource.tofu_provider
+        provider.respond_to?(:disk_renderer_collection?) && provider.disk_renderer_collection?
+      rescue StandardError
+        false
+      end
     end
   end
 end

data/app/models/concerns/foreman_opentofu/compute_attribute_validation.rb

@@ -0,0 +1,25 @@
+module ForemanOpentofu
+  module ComputeAttributeValidation
+    extend ActiveSupport::Concern
+
+    included do
+      validate :validate_opentofu_vm_attributes
+    end
+
+    private
+
+    def validate_opentofu_vm_attributes
+      return unless compute_resource.is_a?(ForemanOpentofu::Tofu)
+
+      args = compute_resource.default_attributes.merge(vm_attrs.deep_dup).to_h.symbolize_keys
+      compute_resource.normalize_vm_args_collections!(args)
+      provider = compute_resource.tofu_provider
+
+      begin
+        provider.validate_vm!(args, compute_resource)
+      rescue ArgumentError => e
+        errors.add(:base, e.message)
+      end
+    end
+  end
+end

data/app/models/concerns/foreman_opentofu/power_commands.rb

@@ -0,0 +1,57 @@
+module ForemanOpentofu
+  module PowerCommands
+    def start_vm(name)
+      return change_vm_power(name, 'on') if tofu_provider.respond_to?(:power_state_attributes)
+
+      output = client({ 'name' => name, 'power_state' => 'on' }).run_create
+      output['vm']['power_state'] == 'on'
+    end
+
+    def stop_vm(name)
+      return change_vm_power(name, 'off') if tofu_provider.respond_to?(:power_state_attributes)
+
+      output = client({ 'name' => name, 'power_state' => 'off' }).run_create
+      output['vm']['power_state'] == 'off'
+    end
+
+    private
+
+    def change_vm_power(identifier, state)
+      vm_command_errors('change power') do
+        tf_state = power_state_record(identifier)
+        output = power_state_output(tf_state)
+        result = client(power_state_arguments(output, tf_state.name, state)).run_create(power_only: true)
+        verify_power_result(result, state)
+      end
+    end
+
+    def power_state_record(identifier)
+      tf_state = TfState.find_by(uuid: identifier) || TfState.find_by(name: identifier)
+      raise "No OpenTofu state found for #{identifier}" unless tf_state
+
+      tf_state
+    end
+
+    def power_state_output(tf_state)
+      output = client('name' => tf_state.name).run_output
+      valid = output['vm'].is_a?(Hash) && output['vm'].present?
+      raise 'Power control requires VM output from the provisioning template.' unless valid
+
+      output
+    end
+
+    def power_state_arguments(output, name, state)
+      arguments = ComputeVM.new(self, output).to_h
+      arguments.merge!(tofu_provider.power_state_attributes(state))
+      arguments['name'] = name
+      arguments
+    end
+
+    def verify_power_result(result, state)
+      vm = ComputeVM.new(self, result)
+      raise "OpenTofu did not return the requested power state #{state}" unless vm.power == state
+
+      vm
+    end
+  end
+end

data/app/models/concerns/foreman_opentofu/vm_command_collection_normalization.rb

@@ -1,7 +1,5 @@
 module ForemanOpentofu
   module VMCommandCollectionNormalization
-    private
-
     def normalize_vm_args_collections!(args)
       args[:image_id] = args[:image] if args.key?(:image)
       normalize_vm_boolean_args!(args)
@@ -13,6 +11,8 @@ module ForemanOpentofu
       end
     end
 
+    private
+
     def normalize_vm_boolean_args!(args)
       vm_boolean_keys.each do |key|
         next unless args.key?(key)
@@ -29,13 +29,38 @@ module ForemanOpentofu
     end
 
     def normalize_collection_input(collection, value)
-      return nested_attributes_for(collection, value) if value.is_a?(Hash) || value.is_a?(ActionController::Parameters)
+      return normalize_nested_hash_collection(collection, value) if value.is_a?(Hash) || value.is_a?(ActionController::Parameters)
 
       Array(value).map do |entry|
         entry.respond_to?(:deep_symbolize_keys) ? entry.deep_symbolize_keys : entry
       end
     end
 
+    def normalize_nested_hash_collection(collection, value)
+      values = (value.respond_to?(:to_unsafe_h) ? value.to_unsafe_h : value.to_h).deep_stringify_keys
+      values.delete("new_#{collection}")
+
+      values.sort_by { |key, _| key.to_s.sub('new_', '').to_i }.filter_map do |_, val|
+        normalized = normalize_nested_collection_value(val)
+        next if normalized.is_a?(Hash) && normalized[:_delete] == '1' && normalized[:id].blank?
+
+        normalized
+      end
+    end
+
+    def normalize_nested_collection_value(value)
+      normalized =
+        if value.is_a?(ActionController::Parameters) && value.respond_to?(:to_unsafe_h)
+          value.to_unsafe_h
+        elsif value.respond_to?(:to_h)
+          value.to_h
+        else
+          value
+        end
+
+      normalized.respond_to?(:deep_symbolize_keys) ? normalized.deep_symbolize_keys : normalized
+    end
+
     def prefill_mandatory_attributes(args)
       # FIXME: add volume/nic attributes
       res = {}

data/app/models/concerns/orchestration/tofu/compute.rb

@@ -6,6 +6,8 @@ module Orchestration
       def match_macs_to_nics(fog_attr)
         return super unless compute_resource.is_a?(ForemanOpentofu::Tofu)
 
+        return super if compute_resource.tofu_provider.respond_to?(:select_nic_for_mac)
+
         interfaces.select(&:physical?).each do |nic|
           mac = vm.send(fog_attr)
           logger.debug "Orchestration::Compute: nic #{nic.inspect} assigned to #{vm.inspect}"

data/app/models/foreman_opentofu/compute_vm.rb

@@ -14,13 +14,24 @@ module ForemanOpentofu
     end
 
     def to_h
-      unwrap(@attributes.to_dup)
+      unwrap(@attributes.deep_dup)
     end
 
     def power
       self['status'] || self['power'] || self['power_state']
     end
 
+    def state
+      power
+    end
+
+    def reload
+      refreshed = @provider.find_vm_by_uuid(self['identity'])
+      @attributes = refreshed.to_h.deep_stringify_keys
+      define_dynamic_readers!
+      self
+    end
+
     # TODO: add definitions for different power on/off values
     def ready?
       power.to_s == 'on' || power.to_s == 'running'
@@ -30,6 +41,21 @@ module ForemanOpentofu
       self['name']
     end
 
+    def vm_description
+      return nil unless @provider.respond_to?(:tofu_provider)
+
+      vm_attrs = @provider.tofu_provider.attributes('vm')
+      values = vm_attrs.filter_map do |attr|
+        key = attr['name'].to_s
+        value = attribute_value(key)
+        next if value.blank?
+
+        label = attr['label'].presence || key.humanize
+        "#{label}: #{value}"
+      end
+      values.join(', ') if values.present?
+    end
+
     def persisted?
       self['identity'].present? || self['id'].present?
     end
@@ -61,6 +87,17 @@ module ForemanOpentofu
       instance_eval(&block)
     end
 
+    # Foreman's MAC orchestration expects NIC objects with attribute readers.
+    def interfaces
+      value = attribute_value('interfaces_attributes') || attribute_value('interfaces') || []
+      value = value.sort_by { |key, _| key.to_i }.map(&:last) if value.is_a?(Hash)
+      Array(value).map { |attrs| OpenStruct.new(attrs) }
+    end
+
+    def select_nic(nics, nic)
+      @provider.tofu_provider.select_nic_for_mac(nics, nic)
+    end
+
     def volumes_attributes
       vols_attrs = attribute_value('volumes_attributes')
       return vols_attrs if vols_attrs.is_a?(Hash)
@@ -119,6 +156,17 @@ module ForemanOpentofu
       end
     end
 
+    def unwrap(value)
+      case value
+      when Hash
+        value.transform_values { |v| unwrap(v) }
+      when Array
+        value.map { |v| unwrap(v) }
+      else
+        value
+      end
+    end
+
     def flatten_attrs(attrs)
       result = {}
 
@@ -144,7 +192,7 @@ module ForemanOpentofu
       key = method_name.to_s
       return attribute_value(key) if dynamic_attribute_keys.include?(key)
 
-      super
+      nil
     end
   end
 end

data/app/models/foreman_opentofu/opentofu_vm_commands.rb

@@ -1,11 +1,14 @@
 module ForemanOpentofu
   module OpentofuVMCommands
     include ForemanOpentofu::VMCommandCollectionNormalization
+    include ForemanOpentofu::PowerCommands
 
     def find_vm_by_uuid(uuid)
       vm_command_errors('find vm') do
         tf_state = ForemanOpentofu::TfState.find_by(uuid: uuid)
         data = client({ 'name' => tf_state&.name }).run_output
+        # trying to add a volume with invalid parameter may persist in the volumes_attributes-hash
+        data['volumes_attributes'] = data['volumes_attributes']&.delete_if { |_idx, vol| vol.key?('id') && vol['id'].nil? }
         ComputeVM.new(self, data)
       end
     end
@@ -15,19 +18,24 @@ module ForemanOpentofu
         args = default_attributes.merge(args).to_h.symbolize_keys
         normalize_vm_args_collections!(args)
         args = prefill_mandatory_attributes(args).merge(args)
+        return ComputeVM.new(self, args.deep_stringify_keys) unless tofu_provider.plan_on_new_vm?
+
         executor = client(args)
         data = executor.run_new
         attrs = data['resource_changes'].first['change']['after'] || {}
-        OpenStruct.new(attrs)
+        attrs = args.deep_stringify_keys.merge(attrs.to_h.deep_stringify_keys)
+        ComputeVM.new(self, attrs)
       end
     end
 
     def create_vm(args = {})
+      args = default_attributes.merge(args).to_h.symbolize_keys
+      normalize_vm_args_collections!(args)
+      tofu_provider.validate_vm!(args, self)
+
       vm_command_errors('create vm') do
-        args = default_attributes.merge(args).to_h.symbolize_keys
-        normalize_vm_args_collections!(args)
         executor = client(args)
-        output = executor.run_create
+        output = executor.run_create(cleanup_on_failure: true)
         ComputeVM.new(self, output)
       end
     end
@@ -41,24 +49,16 @@ module ForemanOpentofu
       tf_state.destroy
     end
 
-    def start_vm(name)
-      output = client({ 'name' => name, 'power_state' => 'on' }).run_create
-      output['vm']['power_state'] == 'on'
-    end
-
-    def stop_vm(name)
-      output = client({ 'name' => name, 'power_state' => 'off' }).run_create
-      output['vm']['power_state'] == 'off'
-    end
-
     def save_vm(uuid, attrs)
       old_attrs = vm_compute_attributes_for(uuid).to_h.deep_stringify_keys
       tf_state = TfState.find_by(uuid: uuid)
       raise StandardError, "VM with UUID #{uuid} does not exist" unless tf_state
+      new_attrs = attrs.to_h.deep_stringify_keys
+      merged_attrs = old_attrs.merge(new_attrs).deep_symbolize_keys
+      normalize_vm_args_collections!(merged_attrs)
+      tofu_provider.validate_vm!(merged_attrs, self)
+
       vm_command_errors('update vm') do
-        new_attrs = attrs.to_h.deep_stringify_keys
-        merged_attrs = old_attrs.merge(new_attrs).deep_symbolize_keys
-        normalize_vm_args_collections!(merged_attrs)
         data = client({ 'name' => tf_state.name }.merge(merged_attrs)).run_create(raise_if_recreate: true)
         ComputeVM.new(self, data)
       end
@@ -84,13 +84,13 @@ module ForemanOpentofu
       yield
     rescue StandardError => e
       Foreman::Logging.exception("Caught #{provider} error", e)
-      raise ::Foreman::WrappedException.new(
-        e,
+      raise ::Foreman::Exception.new(
         N_(
-          "Foreman could not find a required %<provider>s resource in #{method_name}. " \
-          'Check if Foreman has the required permissions and the resource exists. Reason: %<error>s'
+          '%<provider>s failed to %<method>s. Reason: %<error>s'
         ),
-        { provider: provider, error: e.message }
+        provider: provider,
+        method: method_name,
+        error: e.message
       )
     end
 

data/app/models/foreman_opentofu/tofu.rb

@@ -21,9 +21,9 @@ module ForemanOpentofu
     include ComputeResourceCaching
     include KeyPairComputeResource
     validates :provider, presence: true, inclusion: { in: %w[Tofu] }
-    validates :url, presence: true
-    validates :user, presence: true
-    validates :password, presence: true
+    validates :url, presence: true, if: -> { mandatory_compute_resource_attribute?('url') }
+    validates :user, presence: true, if: -> { mandatory_compute_resource_attribute?('user') }
+    validates :password, presence: true, if: -> { mandatory_compute_resource_attribute?('password') }
 
     # alias_attribute :username, :user
     # alias_attribute :endpoint, :url
@@ -67,9 +67,11 @@ module ForemanOpentofu
     end
 
     def opentofu_template
-      return ProvisioningTemplate.find(attrs[:opentofu_template_id]) if attrs.key? :opentofu_template_id
-      # or default-template for this opentofu_provider
-      nil
+      return ProvisioningTemplate.find(attrs[:opentofu_template_id]) if attrs[:opentofu_template_id].present?
+
+      ProvisioningTemplate.unscoped.find_by!(
+        name: tofu_provider.default_template
+      )
     end
 
     def opentofu_template_id
@@ -148,6 +150,12 @@ module ForemanOpentofu
 
     private
 
+    def mandatory_compute_resource_attribute?(name)
+      tofu_provider&.connection_attrs&.any? do |attribute|
+        attribute['name'] == name && attribute['mandatory']
+      end
+    end
+
     def set_vm_volumes_attributes(vm, vm_attrs)
       volumes = if vm.respond_to?(:volumes_attributes)
                   vm.volumes_attributes.values