foreman_opentofu 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13aa21537aa49784d6b945e5e0169c20509c46e361e951b6db79a17d330559ef
-  data.tar.gz: e0f23b3286a236bd161421d8464518a9b47878d61e7ebd03ab41b361cb0b513f
+  metadata.gz: aac98f7ab29438afb053493e3ef572e5374d5140eec49cc67e4f4d9787776a08
+  data.tar.gz: 85f7eb01a3b220b06b357c5ef9b0270bbf49fc19ea286f7e421964871f7bf726
 SHA512:
-  metadata.gz: f1459d8aa7ad4e99b3d43b2725093f7cdc06d7473073432ca8386f46b1d2db4a5ac3acad3ec7c3580830e2d13a11e25354713c9a702a1f3ac669a527a504137f
-  data.tar.gz: 4724e84ebc240ef60bdd95c65710fc3156f718628d38ba65570d8305df762be8883e00df13783002927589cdb2e7eed0304b98b414e5ac4e1edf9a48c5ca3d8c
+  metadata.gz: 3a8231fb351a2f809d768880ed32dd0be78b13333453f4d5fbb71fbcc4c801dc963c0b08beeed69ba916c307c669856d17e8513d1cdb4923f25e0c6b49487e73
+  data.tar.gz: c221f8852cfedf5d07bb40bb93832c2512b77b59b58c043dd8f23cfdf61ee102321be84825158e61981bcd1255b4bc2b6c18fa589d368c5c0d2df16dc2697994

data/README.md

@@ -337,6 +337,23 @@ bundle exec foreman start
 
 See details in [foreman plugin development](https://projects.theforeman.org/projects/foreman/wiki/How_to_Create_a_Plugin)
 
+
+### Testing
+
+#### Template Snapshots
+
+The OpenTofu templates are tested against recorded snapshots similar to the Provisioning Templates in Foreman.
+The snapshots are generated by rendering the templates with predefined fixture-data using the following command in the foreman-directory:
+
+```shell
+bundle exec rake foreman_opentofu:snapshots:generate RAILS_ENV=test
+```
+
+At the moment the fixture-data is defined in `app/services/foreman_opentofu/template_snapshot_service.rb`.
+
+Templates to test must be added to `test/fixtures/renderer/snapshots.yaml`.
+
+
 ## Contributing
 
 Fork and send a Pull Request or create Issue. Thank you.

data/app/controllers/api/v2/tf_states_controller.rb

@@ -2,10 +2,8 @@ module Api
   module V2
     class TfStatesController < ::Api::V2::BaseController
       include ::Api::Version2
+      include ::Foreman::Controller::FilterParameters
 
-      # TODO: verify this
-      # We don't require any of these methods for provisioning
-      # skip_before_action :require_login, :check_user_enabled, :session_expiry, :update_activity_time, :set_taxonomy, :authorize, unless: -> { preview? }
       skip_before_action :set_taxonomy
 
       # Allow HTTP POST methods without CSRF
@@ -14,6 +12,11 @@ module Api
       # overwrite authorize with the local token-based authorization
       before_action :authorize, except: [:destroy]
 
+      # Don't log tfstate response in logs as it can have sensitive information
+      skip_after_action :log_response_body
+
+      filter_parameters :version, :terraform_version, :serial, :lineage, :outputs, :resources, :tf_state, :check_results
+
       resource_description do
         api_version 'v2'
         api_base_url '/foreman_opentofu/api'
@@ -22,15 +25,13 @@ module Api
       def show
         state = ForemanOpentofu::TfState.find_by(name: params[:name])
 
-        if state
-          render plain: state.state, content_type: 'application/json'
-        else
-          render plain: '', status: :not_found
-        end
+        return head :not_found unless state
+
+        render plain: state.state, content_type: 'application/json'
       end
 
       def create
-        raw_state = request.body.read
+        raw_state = request.raw_post
         if raw_state.blank?
           render plain: 'Missing state body', status: :unprocessable_entity
           return
@@ -41,10 +42,9 @@ module Api
           state = ForemanOpentofu::TfState.find_or_initialize_by(name: params[:name])
           state.state = raw_state
           state.save!
-          render plain: '', status: :ok
-        rescue JSON::ParserError => e
-          Rails.logger.error("Invalid state JSON: #{e.message}")
-          render plain: 'Invalid state format', status: :unprocessable_entity
+          head :ok
+        rescue JSON::ParserError
+          render plain: 'Invalid state format', status: :bad_request
         end
       end
 

data/app/lib/foreman_opentofu/concerns/base_template_scope_extensions.rb

@@ -97,6 +97,8 @@ module ForemanOpentofu
       end
 
       def build_disks
+        return rendered_disk(0, disk_render_source.first).to_s if disk_renderer_collection?
+
         disk_render_source.each_with_index.filter_map do |disk, index|
           # drop removed disks; tofu will drop them automatically, if they are no longer defined
           next if disk.respond_to?(:[]) && disk['_delete'].to_i == 1
@@ -151,6 +153,15 @@ module ForemanOpentofu
           to_hcl(data, snippet: true)
         end
       end
+
+      def disk_renderer_collection?
+        return false unless @compute_resource.respond_to?(:tofu_provider)
+
+        provider = @compute_resource.tofu_provider
+        provider.respond_to?(:disk_renderer_collection?) && provider.disk_renderer_collection?
+      rescue StandardError
+        false
+      end
     end
   end
 end

data/app/models/concerns/foreman_opentofu/vm_command_collection_normalization.rb

@@ -29,13 +29,38 @@ module ForemanOpentofu
     end
 
     def normalize_collection_input(collection, value)
-      return nested_attributes_for(collection, value) if value.is_a?(Hash) || value.is_a?(ActionController::Parameters)
+      return normalize_nested_hash_collection(collection, value) if value.is_a?(Hash) || value.is_a?(ActionController::Parameters)
 
       Array(value).map do |entry|
         entry.respond_to?(:deep_symbolize_keys) ? entry.deep_symbolize_keys : entry
       end
     end
 
+    def normalize_nested_hash_collection(collection, value)
+      values = (value.respond_to?(:to_unsafe_h) ? value.to_unsafe_h : value.to_h).deep_stringify_keys
+      values.delete("new_#{collection}")
+
+      values.sort_by { |key, _| key.to_s.sub('new_', '').to_i }.filter_map do |_, val|
+        normalized = normalize_nested_collection_value(val)
+        next if normalized.is_a?(Hash) && normalized[:_delete] == '1' && normalized[:id].blank?
+
+        normalized
+      end
+    end
+
+    def normalize_nested_collection_value(value)
+      normalized =
+        if value.is_a?(ActionController::Parameters) && value.respond_to?(:to_unsafe_h)
+          value.to_unsafe_h
+        elsif value.respond_to?(:to_h)
+          value.to_h
+        else
+          value
+        end
+
+      normalized.respond_to?(:deep_symbolize_keys) ? normalized.deep_symbolize_keys : normalized
+    end
+
     def prefill_mandatory_attributes(args)
       # FIXME: add volume/nic attributes
       res = {}

data/app/models/foreman_opentofu/compute_vm.rb

@@ -14,7 +14,7 @@ module ForemanOpentofu
     end
 
     def to_h
-      unwrap(@attributes.to_dup)
+      unwrap(@attributes.deep_dup)
     end
 
     def power
@@ -30,6 +30,21 @@ module ForemanOpentofu
       self['name']
     end
 
+    def vm_description
+      return nil unless @provider.respond_to?(:tofu_provider)
+
+      vm_attrs = @provider.tofu_provider.attributes('vm')
+      values = vm_attrs.filter_map do |attr|
+        key = attr['name'].to_s
+        value = attribute_value(key)
+        next if value.blank?
+
+        label = attr['label'].presence || key.humanize
+        "#{label}: #{value}"
+      end
+      values.join(', ') if values.present?
+    end
+
     def persisted?
       self['identity'].present? || self['id'].present?
     end
@@ -119,6 +134,17 @@ module ForemanOpentofu
       end
     end
 
+    def unwrap(value)
+      case value
+      when Hash
+        value.transform_values { |v| unwrap(v) }
+      when Array
+        value.map { |v| unwrap(v) }
+      else
+        value
+      end
+    end
+
     def flatten_attrs(attrs)
       result = {}
 
@@ -144,7 +170,7 @@ module ForemanOpentofu
       key = method_name.to_s
       return attribute_value(key) if dynamic_attribute_keys.include?(key)
 
-      super
+      nil
     end
   end
 end

data/app/models/foreman_opentofu/opentofu_vm_commands.rb

@@ -18,7 +18,8 @@ module ForemanOpentofu
         executor = client(args)
         data = executor.run_new
         attrs = data['resource_changes'].first['change']['after'] || {}
-        OpenStruct.new(attrs)
+        attrs = args.deep_stringify_keys.merge(attrs.to_h.deep_stringify_keys)
+        ComputeVM.new(self, attrs)
       end
     end
 

data/app/services/foreman_opentofu/opentofu_executer.rb

@@ -80,10 +80,14 @@ module ForemanOpentofu
           tofu.plan
           raise 'OpenTofu planned to re-create a resource; action aborted (check logs for details)!' if plan_wants_recreate? tofu.show_plan
         end
-        tofu.apply
-        attrs = tofu.output('vm_attrs')
-        ForemanOpentofu::TfState.find_by(name: @cr_attrs['name'])&.update(uuid: attrs['identity'])
-        attrs
+        begin
+          tofu.apply
+          attrs = tofu.output('vm_attrs')
+          ForemanOpentofu::TfState.find_by(name: @cr_attrs['name'])&.update(uuid: attrs['identity'])
+          attrs
+        rescue StandardError => e
+          handle_failed_create(tofu, e)
+        end
       end
     end
 
@@ -177,5 +181,22 @@ module ForemanOpentofu
       end
       false
     end
+
+    def handle_failed_create(tofu, error)
+      tofu.destroy
+      ForemanOpentofu::TfState.find_by(name: @cr_attrs['name'])&.destroy
+      raise error
+    rescue StandardError => e
+      raise if e.equal?(error)
+
+      Rails.logger.error(
+        "Removing OpenTofu resource after host create for #{@cr_attrs['name']} failed: #{e.message}"
+      )
+      wrapped_error = error.exception(
+        "#{error.message}\n Removing resource after failed host creation also failed: #{e.message}"
+      )
+      wrapped_error.set_backtrace(error.backtrace)
+      raise wrapped_error
+    end
   end
 end

data/app/services/foreman_opentofu/provider_type.rb

@@ -119,5 +119,11 @@ module ForemanOpentofu
 
       result
     end
+
+    # Whether disk renderer emits one collection resource for all disks (for_each)
+    # instead of one resource block per disk entry.
+    def disk_renderer_collection?
+      false
+    end
   end
 end

data/app/services/foreman_opentofu/renderer/source/snapshot.rb

@@ -0,0 +1,25 @@
+module ForemanOpentofu
+  module Renderer
+    module Source
+      class Snapshot < Foreman::Renderer::Source::Snapshot
+        SNAPSHOTS_DIRECTORY = ForemanOpentofu::Engine.root.join('test', 'fixtures', 'renderer', 'snapshots')
+
+        class << self
+          def snapshot_path(template, suffix = 'host4dhcp')
+            File.join(SNAPSHOTS_DIRECTORY, "#{template_path(template).tr(' ', '_')}.#{suffix}.snap.txt")
+          end
+
+          def hosts(template)
+            ['empty', "#{template.name.split(' ').first.downcase}_host"]
+          end
+        end
+
+        private
+
+        def templates_directory
+          ForemanOpentofu::TemplateSnapshotService::TEMPLATES_DIRECTORY
+        end
+      end
+    end
+  end
+end

data/app/services/foreman_opentofu/template_snapshot_service.rb

@@ -0,0 +1,68 @@
+module ForemanOpentofu
+  # < Foreman::TemplateSnapshotService
+  class TemplateSnapshotService
+    TEMPLATES_DIRECTORY = ForemanOpentofu::Engine.root.join('app', 'views', 'templates', 'provisioning')
+
+    def self.templates
+      new.templates
+    end
+
+    def self.render_template(template, host_name = :empty)
+      raise 'TemplateSnapshotService can only be used in test environment' unless Rails.env.test?
+      require 'factory_bot_rails'
+
+      cr_attrs = new.send(host_name.to_sym)
+      provider_type = template.name.split(' ').first.downcase
+      variables = {
+        compute_resource: FactoryBot.build("opentofu_#{provider_type}_cr".to_sym),
+        cr_attrs: cr_attrs || {},
+        use_backend: true,
+        token: 'very_secret_token',
+        host_name: host_name,
+        resource: nil,
+      }
+      variables[:ssh_key] = variables[:compute_resource].key_pair if variables[:compute_resource].key_pair
+      source = ForemanOpentofu::Renderer::Source::Snapshot.new(template)
+      scope = Foreman::Renderer.get_scope(source: source, variables: variables)
+      Foreman::Renderer::UnsafeModeRenderer.render(source, scope)
+    end
+
+    def templates
+      files.map { |path| ForemanOpentofu::Renderer::Source::Snapshot.load_file(path) }
+    end
+
+    def hetzner_host
+      {
+        server_type: 'cx23',
+      }
+    end
+
+    def nutanix_host
+      {
+        'cluster_uuid' => 'my_cluster_uuid',
+        'volumes_attributes' => [
+          {
+            size_gb: 40,
+            container_uuid: 'my_container_uuid',
+          },
+        ],
+        'interfaces_attributes' => [
+          {
+            model: 'E1000',
+            subnet_uuid: 'my_subnet_uuid',
+          },
+        ],
+      }
+    end
+
+    def empty
+      {}
+    end
+
+    private
+
+    def files
+      @files ||= YAML.safe_load_file(ForemanOpentofu::Engine.root.join('test', 'fixtures', 'renderer', 'snapshots.yaml')).fetch('files', []).map { |path| File.join(TEMPLATES_DIRECTORY, path) }
+    end
+  end
+end

data/app/views/api/v2/compute_resources/tofu.json.rabl

@@ -0,0 +1 @@
+attributes :user, :opentofu_provider, :opentofu_template_id

data/app/views/compute_resources_vms/form/tofu/_network.html.erb

@@ -1 +1,4 @@
-<%= render :partial => provider_partial(compute_resource, "dynamic_attrs"), :locals => { f: f, attrs: compute_resource.tofu_provider.attributes('nic'), compute_resource: compute_resource } %>
+<% index = f.index.present? ? f.index : 0 %>
+<%= field_set_tag _("Network"), :id => "network_#{index}" do %>
+  <%= render :partial => provider_partial(compute_resource, "dynamic_attrs"), :locals => { f: f, attrs: compute_resource.tofu_provider.attributes('nic'), compute_resource: compute_resource } %>
+<% end %>

data/app/views/compute_resources_vms/form/tofu/_volume.html.erb

@@ -1,2 +1,4 @@
-<%= render :partial => provider_partial(compute_resource, "dynamic_attrs"), :locals => { f: f, attrs: compute_resource.tofu_provider.attributes('disk'), compute_resource: compute_resource } %>
-
+<% index = f.index.present? ? f.index : 0 %>
+<%= field_set_tag _("Volume"), :id => "volume_#{index}" do %>
+  <%= render :partial => provider_partial(compute_resource, "dynamic_attrs"), :locals => { f: f, attrs: compute_resource.tofu_provider.attributes('disk'), compute_resource: compute_resource } %>
+<% end %>

data/app/views/foreman_opentofu/compute_resources_vms/form/tofu/_interfaces_fields.html.erb

@@ -5,6 +5,7 @@
   interfaces = [compute_resource.new_interface].compact if interfaces.empty?
 %>
 <% interfaces.each_with_index do |interface_obj, idx| %>
+  <% interface_obj = compute_resource.new_interface(interface_obj.to_h) if interface_obj.is_a?(Hash) %>
   <%= fields_for "#{f.object_name}[#{association_attributes}][#{idx}]", interface_obj do |i| %>
     <%= render :partial => provider_partial(compute_resource, 'network'),
                :locals => { :f => i, :compute_resource => compute_resource, :new_host => new_host, :new_vm => new_vm, :remove_title => _('remove network interface'), :selected_cluster => selected_cluster },

data/app/views/foreman_opentofu/compute_resources_vms/form/tofu/_volumes_fields.html.erb

@@ -1,5 +1,12 @@
 <%
-  volumes = f.object.volumes_attributes
+  volumes =
+    if f.object.respond_to?(:volumes_attributes)
+      f.object.volumes_attributes
+    elsif f.object.respond_to?(:volumes)
+      f.object.volumes
+    else
+      {}
+    end
   volumes =
     if volumes.is_a?(Hash)
       volumes.sort_by { |key, _| key.to_s.sub('new_', '').to_i }

data/app/views/templates/provisioning/hetzner_provision_host.erb

@@ -30,11 +30,9 @@ data "hcloud_ssh_key" "deploy_key" {
 }
 <%- end %>
 
-# Create a new server running debian
+# Create a new server
 resource "hcloud_server" "node1" {
   name        = "<%= @host_name %>"
-  # image       = "debian-12"
-  # server_type = "cx23"
 
   <%= vm_attributes(['name', 'image']) %>
 
@@ -68,27 +66,14 @@ output "vm_attrs" {
       vm_ip6_address = hcloud_server.node1.ipv6_address
       vm = hcloud_server.node1
       volumes_attributes = try({
-        for v in values(hcloud_volume.volumes) : tostring(v.id) => {
-          id = v.id
-          name = v.name
-          size = v.size
+        for k, v in try(local.disks, {}) : tostring(try(v.id, k)) => {
+          id = try(v.id, null)
+          name = try(v.name, "volume-${k}")
+          size = try(v.size, 20)
           automount = try(v.automount, null)
           format = try(v.format, null)
-        }
+        } if try(v["_delete"], "0") != "1"
       }, {})
 }
 }
-
-<% else %>
-# # get available images
-# data "hcloud_images" "all" {}
-# output "images" {
-#   value = data.hcloud_images.all.images
-# }
-#
-# # get available server types
-# data "hcloud_server_types" "all" {}
-# output "server_types" {
-#   value = data.hcloud_server_types.all.server_types
-# }
 <% end %>

data/lib/foreman_opentofu/engine.rb

@@ -47,6 +47,7 @@ module ForemanOpentofu
     # Include concerns in this config.to_prepare block
     config.to_prepare do
       ::ComputeResourcesController.include ForemanOpentofu::Controller::Parameters::ComputeResource
+      ::Api::V2::ComputeResourcesController.include ForemanOpentofu::Controller::Parameters::ComputeResource
       ::ComputeResourcesVmsController.include ForemanOpentofu::ComputeResourcesVmsController
       ::Host::Managed.include Orchestration::Tofu::Compute
       ::Authorizer.prepend ForemanOpentofu::AuthorizerPowerOverride unless ::Authorizer < ForemanOpentofu::AuthorizerPowerOverride

data/lib/foreman_opentofu/provider_types/hetzner.rb

@@ -5,6 +5,10 @@ require "#{ForemanOpentofu::Engine.root}/app/services/foreman_opentofu/provider_
 ForemanOpentofu::ProviderTypeManager.register('hetzner') do
   @capabilities = [:image, :key_pair, :power_status_only]
 
+  def disk_renderer_collection?
+    true
+  end
+
   def provided_attributes
     {
       ip: :vm_ip_address,
@@ -125,7 +129,7 @@ ForemanOpentofu::ProviderTypeManager.register('hetzner') do
       }
 
       resource "hcloud_volume" "volumes" {
-        for_each  = { for k, d in local.disks : tostring(try(d.id, k)) => d if try(d["_delete"], "0") != "1" }
+        for_each  = { for k, d in local.disks : tostring(k) => d if try(d["_delete"], "0") != "1" }
         name      = try(each.value.name, "volume-${each.key}")
         size      = try(each.value.size, 20)
         server_id = hcloud_server.node1.id

data/lib/foreman_opentofu/provider_types/nutanix.rb

@@ -55,7 +55,7 @@ ForemanOpentofu::ProviderTypeManager.register('nutanix') do
   self.disk_renderer = proc do |disk|
     {
       disk_list: {
-        disk_size_mib: disk[:size_gb] * 1024,
+        disk_size_mib: disk[:size_gb].to_i * 1024,
         storage_config: {
           storage_container_reference: {
             kind: 'storage_container',

data/lib/foreman_opentofu/version.rb

@@ -1,3 +1,3 @@
 module ForemanOpentofu
-  VERSION = '0.0.5'.freeze
+  VERSION = '0.0.6'.freeze
 end

data/lib/tasks/snapshots.rake

@@ -0,0 +1,63 @@
+namespace :foreman_opentofu do
+  namespace :snapshots do
+    desc 'Generate snapshots'
+    task generate: :environment do
+      unless Rails.env.test?
+        puts 'This task can only be run in test environment'
+        exit
+      end
+
+      require 'database_cleaner'
+      require 'factory_bot_rails'
+
+      # Add plugin to FactoryBot's paths
+      FactoryBot.definition_file_paths << ForemanOpentofu::Engine.root.join('test', 'factories')
+      FactoryBot.reload
+
+      module BaseMacrosStub
+        def dns_lookup(_name_or_ip)
+          'foreman.example.com'
+        end
+      end
+
+      # ::Foreman::Plugin.singleton_class.send :prepend, PluginSnapshotStub
+      ::Foreman::Renderer::Scope::Base.prepend BaseMacrosStub
+
+      # clean the snapshot directory in order to delete renamed ones and keep it clean
+      FileUtils.rm_rf(Dir.glob(File.join(
+        ::ForemanOpentofu::Renderer::Source::Snapshot::SNAPSHOTS_DIRECTORY, '*'
+      )))
+
+      DatabaseCleaner.cleaning do
+        Foreman.settings.load
+        Setting[:foreman_url] = 'http://foreman.example.com'
+
+        User.current = FactoryBot.build(:user, :admin)
+        admin = FactoryBot.create(:user, :admin, password: 'password123', auth_source: FactoryBot.create(:auth_source_ldap))
+
+        failed_snapshots = []
+
+        User.as(admin.login) do
+          ForemanOpentofu::TemplateSnapshotService.templates.each do |template|
+            ForemanOpentofu::Renderer::Source::Snapshot.hosts(template).each do |host|
+              snapshot_path = ForemanOpentofu::Renderer::Source::Snapshot.snapshot_path(template, host)
+              dir = File.dirname(snapshot_path)
+              FileUtils.mkdir_p(dir) unless File.directory?(dir)
+
+              begin
+                snapshot = ForemanOpentofu::TemplateSnapshotService.render_template(template, host)
+                puts "Writing #{snapshot_path}"
+                File.write(snapshot_path, snapshot)
+              rescue StandardError => e
+                warn "Snapshot #{snapshot_path} failed with #{e}"
+                failed_snapshots << snapshot_path
+              end
+            end
+          end
+        end
+
+        raise "#{failed_snapshots.count} snapshots failed generation or validation. Please inspect the output to see the details." unless failed_snapshots.empty?
+      end
+    end
+  end
+end