foreman_opentofu 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6bff811eebb374eb3983de89c83faf991c8021e416c677bbfda90837cbeeb5a
-  data.tar.gz: 8741e8b977ba0a078c13ac4642a790e11646f719358f22b69c299368d698a8f0
+  metadata.gz: ccfa3742aa9fe09edc20f685bf1a65f0f32aa34603572812fd250f64f84c7f60
+  data.tar.gz: 632627c1dbecb3da3282fa29085647263f588eb99753db33d2f2ada53b57680c
 SHA512:
-  metadata.gz: 67e87113d04444061daa977fca5721d26122035fd821f86284667eb5ad247c7a2002d5bd1d451ec0d4cea84ade9f530eb1dcffa3afb6b931b7a0b78f1f57c5ca
-  data.tar.gz: d2627a46149453ae41fa6f5463bd792501ee51bc0760693a6bafdd3f0da7357ab9e8b8093c98149b5c4b4c7b2e8e03698c8d97996bb1ec3d1ac0990bc719a455
+  metadata.gz: 1858acdfd4b8e7e32dd3c91765ca1e5e6921da70b8754842e1fa8284bb9b1ceb23b5abb7b22320218d6326167f821d032071915458155545e19ace88dfadd996
+  data.tar.gz: a9f29bd6e263acb43c2a692ed848a71b99c61dce3fb3b64a9ebd7597bceae0280f8b93879b2e5b89a6a8a2244f9cfd5644fdfdc20444353fc962211def091591

data/README.md

@@ -90,6 +90,73 @@ A short config file might look like this:
 
 The name of the file must be the same as the provider-type name we set in the next step (e.g. `/config/nutanix.json`).
 
+##### Dynamic Config Parameter
+
+Sometimes it is necessary to provide a list of possible values that are defined by the backend-service.
+Curating the 'options'-Array is tedious at best or not possible if multiple instances of the backend service are in use.
+This can be addressed by specifiying an OpenTofu-Provider's [DataSource](https://opentofu.org/docs/language/data-sources/) in the following way:
+
+```json
+{
+  "name": "volume_group", "type": "select", "group": "disk", "mandatory": true, "label": "Volume Group",
+  "options": {
+    "data_source": {
+      "name": "nutanix_volume_groups_v2",
+      "arguments": {
+        "filter": "name eq 'volume_group_test'",
+        "limit": 20
+      },
+      "entity": {
+        "id": "metadata.uuid"
+      }
+    },
+    "output_path_postfix": "volume_groups"
+  }
+}
+
+```
+The GUI requires a list of objects that at least contain a name and an id for each select-option.
+The `entity` section can be used to define a specific value from an object within the list that the DataSource returns.
+If the object already has `name` and `id` entries, these will automatically used.
+In the above example `name` exists in the object and can be used.
+For the `id` however, a different value must be selected from the object.
+
+This requests the data via OpenTofu in the following construct:
+
+```hcl
+data "nutanix_volume_groups_v2" "all" {
+  filter = "name eq 'volume_group_test'"
+  limit = 20
+}
+output "resources" {
+  value = [ for e in data.nutanix_volume_groups_v2.all.volume_groups: {
+    id = e.metadata.uuid
+    name = e.name
+  } ]
+}
+```
+
+##### Special Parameter
+
+Some config parameter names have special meanings.
+For instance, Image-based Deployment requires binding images available on the backend-service with Operating Systems configured in Foreman.
+To enable Foreman OpenTofu to display the available images, a `select`-parameter with the name `available_images` must be specified.
+It is recommended to tie this to a data-source available in the OpenTofu provider.
+
+```json
+{
+  "name": "available_images", "type": "select",
+  "options": {
+    "data_source": {
+      "name": "hcloud_images",
+      "arguments": { "with_architecture": ["x86"] }
+    },
+    "output_path_postfix": "images"
+  }
+}
+```
+
+
 #### Create Provider Type
 
 To let the Foreman OpenTofu Plugin know about your new Provider Type, one additional file has to be created in `/lib/foreman_opentofu/provider_types/`.

data/app/lib/foreman_opentofu/concerns/base_template_scope_extensions.rb

@@ -3,19 +3,65 @@ module ForemanOpentofu
     module BaseTemplateScopeExtensions
       extend ActiveSupport::Concern
       extend ApipieDSL::Module
+      include ForemanOpentofu::HclFormat
+      include ForemanOpentofu::NicHelpers
 
       apipie :class, 'Base macros related to Opentofu templates' do
-        name 'Base Content'
-        sections only: %w[all provisioning]
+        name 'OpenTofu helpers'
+        sections only: %w[opentofu_script]
+      end
+
+      apipie :method, 'Returns `terraform`-block including necessary backend definition, if applicable' do
+        required :data, Hash, desc: 'Define the provider-type to pull in, e.g. `{ \'nutanix\' => { \'source\' => \'nutanix/nutanix\', \'version\' => \'2.4.0\' }`'
+        returns String, desc: '`terraform {}`-block based on the data-input, if applicable with TfState-Backend definition.'
+      end
+      # e.g. terraform_block({ 'nutanix' => { 'source' => 'nutanix/nutanix', 'version' => '2.4.0' })
+      def terraform_block(data)
+        block = block_to_hcl(['terraform'])
+        block << '{'
+        block << block_to_hcl(['required_providers'], data, depth: 1)
+        block << backend_block
+        block << "\n}"
+      end
+
+      apipie :method, 'Add "resource" data_source block' do
+        returns String, desc: ''
+      end
+      def resource_block(resource)
+        block = ''
+        path = ['data', resource[:name], 'all']
+
+        # data "<%= @resource[:name] %>" "all" {
+        # <% @resource.dig(:options, 'data_source', 'arguments')&.each do |key, value| %>
+        #   <%= key %> = <%= value.inspect %>
+        # <% end %>
+        # }
+        block << block_to_hcl(path, resource.dig(:options, 'data_source', 'arguments') || {})
+
+        # output "resources" {
+        #   value = [ for e in data.<%= @resource[:name] %>.all.<%= @resource.dig(:options, 'output_path_postfix') %>: {
+        #     id = e.<%= @resource.dig(:options, 'entity', 'id') || 'id' %>
+        #     name = e.<%= @resource.dig(:options, 'entity', 'name') || 'name' %>
+        #     # obj = e
+        #     } ]
+        # }
+        block << block_to_hcl(%w[output resources])
+        block << '{' << "\n"
+        block << "  value = [ for e in data.#{resource[:name]}.all.#{resource.dig(:options, 'output_path_postfix')}: {\n"
+        block << "    id = e.#{resource.dig(:options, 'entity', 'id') || 'id'}\n"
+        block << "    name = e.#{resource.dig(:options, 'entity', 'name') || 'name'}\n"
+        # block << 'obj = e'
+        block << '  } ]' << "\n"
+        block << '}' << "\n"
       end
 
       apipie :method, 'Returns all VM parameters' do
         required :skip_list, Array, desc: 'List of parameters to skip'
         returns String, desc: '"key = value" lines'
       end
-
       def vm_attributes(skip_list = [])
         available_attributes = @compute_resource.available_attributes
+        data = {}
         res = ''
         @cr_attrs.each do |key, value|
           next if skip_list.include? key
@@ -28,60 +74,22 @@ module ForemanOpentofu
           next if conf['group'] != 'vm'
           next if value.blank? && !conf['mandatory']
 
-          res << "#{key} = #{format_value(value, conf['type'])}\n"
-        end
-        res << nic_attributes(available_attributes)
-      end
-
-      def nic_attributes(available_attributes)
-        interfaces = @cr_attrs['interfaces'] || @cr_attrs['interfaces_attributes']
-        return '' if interfaces.blank?
-
-        interfaces = normalize_interfaces(interfaces)
-        nic_defs = available_attributes.values.select do |attrs|
-          attrs['group'] == 'nic'
-        end
-        res = ''
-        interfaces.each do |iface|
-          next if iface['subnet_uuid'].blank?
-
-          res << build_attribute_block('nic_list', iface, nic_defs)
+          data[key] = format_value(value, conf['type'])
         end
-        res
+        res << to_hcl(data, snippet: true)
       end
 
-      def normalize_interfaces(interfaces)
-        if interfaces.is_a?(Hash)
-          if interfaces.keys.all? { |k| k.to_s =~ /^\d+$/ }
-            interfaces.values
-          else
-            [interfaces]
-          end
+      def backend_block
+        if @token
+          data = {
+            address: "#{Setting[:foreman_url]}/api/v2/tf_states/#{@host_name}",
+            headers: {
+              'Authorization' => "Token #{@token}",
+            },
+          }
+          block_to_hcl(%w[backend http], data, depth: 1)
         else
-          Array(interfaces)
-        end
-      end
-
-      def build_attribute_block(block_name, attrs, nic_defs)
-        res = "#{block_name} {\n"
-        attrs.each do |k, v|
-          next if v.blank?
-          conf = nic_defs.find { |a| (a['name'] || a[:name]) == k }
-          next unless conf
-          res << "  #{k} = #{format_value(v, conf['type'])}\n" if conf
-        end
-        res << "}\n"
-        res
-      end
-
-      private
-
-      def format_value(val, type)
-        case type
-        when 'string', 'select' then "\"#{val}\""
-        when 'bool' then Foreman::Cast.to_bool(val)
-        when 'number' then val.to_i
-        else val
+          ''
         end
       end
     end

data/app/lib/foreman_opentofu/hcl_format.rb

@@ -0,0 +1,95 @@
+module ForemanOpentofu
+  module HclFormat
+    def default_opts(opts = {})
+      {
+        indent: 2,
+        depth: 0,
+        snippet: false,
+      }.merge opts
+    end
+
+    # possible `opts`:
+    #   `indent` : number of whitespace to indent; default 2
+    #   `depth`  : start value of depth (for indentation); default: 0
+    #   `snippet`: if `true` and var is a `Hash` string will not be framed with `{}`
+    def to_hcl(var, opts = {})
+      opts = default_opts.merge(opts)
+
+      case var
+      when Hash then hash_to_hcl(var, opts)
+      when Array then array_to_hcl(var, opts)
+      when String then var.inspect
+      else var.to_s
+      end
+    end
+
+    def prefix_hcl(opts)
+      "\n#{' ' * opts[:indent] * opts[:depth]}"
+    end
+
+    # output a hcl-block:
+    # hello "how" "are" "you" { ... }
+    # the above would be created by: block_to_hcl(['hello,'how', 'are', 'you'], { .. })
+    def block_to_hcl(names, content = nil, opts = {})
+      opts = default_opts(opts)
+
+      hcl = prefix_hcl(opts)
+      hcl << names[0]
+      hcl << ' '
+      # sub-block-names are quoted
+      hcl << names[1..].map(&:to_s).map(&:inspect).join(' ')
+      hcl << ' ' if hcl[-1] != ' '
+      hcl << to_hcl(content, opts)
+    end
+
+    def hash_to_hcl(hsh, opts)
+      opts = default_opts(opts)
+      hcl = ''
+      new_opts = opts.merge(snippet: false)
+
+      unless opts[:snippet]
+        hcl << '{'
+        new_opts[:depth] = opts[:depth] + 1
+      end
+
+      close_block_on_newline = false
+
+      hsh.each do |key, value|
+        hcl << prefix_hcl(new_opts)
+        hcl << "#{key} = #{to_hcl(value, new_opts)}"
+        close_block_on_newline = true
+      end
+      hcl << prefix_hcl(opts) if close_block_on_newline
+      hcl << '}' unless opts[:snippet]
+      hcl
+    end
+
+    def array_to_hcl(hsh, opts)
+      opts = default_opts(opts)
+      hcl = '['
+      new_opts = opts.merge(
+        depth: opts[:depth] + 1
+      )
+      close_block_on_newline = false
+
+      hsh.each do |value|
+        hcl << prefix_hcl(new_opts)
+        hcl << to_hcl(value, new_opts)
+        hcl << ','
+        close_block_on_newline = true
+      end
+      hcl.chomp!(',')
+      hcl << prefix_hcl(opts) if close_block_on_newline
+      hcl << ']'
+    end
+
+    def format_value(val, type)
+      case type
+      when 'string', 'select' then val
+      when 'bool' then Foreman::Cast.to_bool(val)
+      when 'number' then val.to_i
+      else val
+      end
+    end
+  end
+end

data/app/lib/foreman_opentofu/nic_helpers.rb

@@ -0,0 +1,47 @@
+module ForemanOpentofu
+  module NicHelpers
+    include ForemanOpentofu::HclFormat
+
+    def nic_attributes(block_name = nil)
+      nic_defs = @compute_resource.available_attributes('nic')
+      interfaces = normalize_interfaces(@cr_attrs['interfaces'] || @cr_attrs['interfaces_attributes'])
+      res = ''
+      interfaces.each do |iface|
+        missing_attrs = nic_defs.select { |name, cfg| cfg[:mandatory] && iface[name].blank? }
+        # TODO: log the fact that we skip this due to missing mandatory attributes
+        next unless missing_attrs.empty?
+
+        res << if block_given?
+                 yield(iface, nic_defs)
+               else
+                 block_to_hcl([block_name], sanitize_attributes(iface, nic_defs), depth: 1)
+               end
+      end
+      res
+    end
+
+    def normalize_interfaces(interfaces)
+      if interfaces.is_a?(Hash)
+        if interfaces.keys.all? { |k| k.to_s =~ /^\d+$/ }
+          interfaces.values
+        else
+          [interfaces]
+        end
+      else
+        Array(interfaces)
+      end
+    end
+
+    def sanitize_attributes(attrs, defs)
+      data = {}
+      attrs.each do |k, v|
+        next if v.blank?
+
+        next unless defs[k]
+
+        data[k] = format_value(v, defs.dig(k, :type))
+      end
+      data
+    end
+  end
+end

data/app/models/concerns/orchestration/tofu/compute.rb

@@ -3,13 +3,9 @@ module Orchestration
     module Compute
       extend ActiveSupport::Concern
 
-      def computeValue(_foreman_attr, fog_attr)
-        value = ''
-        value += vm.send(fog_attr).to_s
-        value
-      end
-
       def match_macs_to_nics(fog_attr)
+        return super unless compute_resource.is_a?(ForemanOpentofu::Tofu)
+
         interfaces.select(&:physical?).each do |nic|
           mac = vm.send(fog_attr)
           logger.debug "Orchestration::Compute: nic #{nic.inspect} assigned to #{vm.inspect}"
@@ -19,6 +15,29 @@ module Orchestration
         end
         true
       end
+
+      def setUserData
+        return super unless compute_resource.is_a?(ForemanOpentofu::Tofu)
+
+        logger.info "Rendering UserData template for #{name}"
+        template = provisioning_template(kind: 'cloud-init')
+        template ||= provisioning_template(kind: 'user_data')
+        # For some reason this renders as 'built' in spoof view but 'provision' when
+        # actually used. For now, use foreman_url('built') in the template
+        if template.nil?
+          # rubocop:disable Layout/LineLength
+          failure(format(_("Image \"%{image}\" needs user data, but \"%{os}\" is not associated to any provisioning template of the kind user_data. Please associate it with a suitable template or uncheck 'User data' from the image definition."),
+            image: image.name,
+            os: operatingsystem))
+          # rubocop:enable Layout/LineLength
+          return false
+        end
+
+        compute_attributes['user_data'] = render_template(template: template)
+
+        return false if errors.any?
+        true
+      end
     end
   end
 end

data/app/models/foreman_opentofu/compute_vm.rb

@@ -47,6 +47,16 @@ module ForemanOpentofu
       reboot
     end
 
+    def vm_ip_address
+      @attributes['vm_ip_address']
+    end
+
+    def wait_for(&block)
+      # TODO: I guess we have nothing to wait for
+      # and we need to change the context of the given block
+      instance_eval(&block)
+    end
+
     private
 
     def define_dynamic_readers!

data/app/models/foreman_opentofu/opentofu_vm_commands.rb

@@ -3,7 +3,7 @@ module ForemanOpentofu
     def find_vm_by_uuid(uuid)
       vm_command_errors('find vm') do
         tf_state = ForemanOpentofu::TfState.find_by(uuid: uuid)
-        data = client({ 'name' => tf_state&.name }).run('output')
+        data = client({ 'name' => tf_state&.name }).run_output
         ComputeVM.new(self, data)
       end
     end
@@ -12,7 +12,7 @@ module ForemanOpentofu
       vm_command_errors('new vm') do
         args = default_attributes.merge(args)
         executor = client(args)
-        data = executor.run('new')
+        data = executor.run_new
         OpenStruct.new(data['resource_changes'].first['change']['after'])
       end
     end
@@ -21,14 +21,14 @@ module ForemanOpentofu
       vm_command_errors('create vm') do
         args = default_attributes.merge(args)
         executor = client(args)
-        output = executor.run('create')
+        output = executor.run_create
         ComputeVM.new(self, output)
       end
     end
 
     def destroy_vm(uuid)
       tf_state = ForemanOpentofu::TfState.find_by(uuid: uuid)
-      client({ 'name' => tf_state&.name }).run('destroy')
+      client({ 'name' => tf_state&.name }).run_destroy
       return unless tf_state
 
       Rails.logger.info "Deleting tfstate for #{tf_state&.name}"
@@ -36,12 +36,12 @@ module ForemanOpentofu
     end
 
     def start_vm(name)
-      output = client({ 'name' => name, 'power_state' => 'on' }).run('create')
+      output = client({ 'name' => name, 'power_state' => 'on' }).run_create
       output['vm']['power_state'] == 'on'
     end
 
     def stop_vm(name)
-      output = client({ 'name' => name, 'power_state' => 'off' }).run('create')
+      output = client({ 'name' => name, 'power_state' => 'off' }).run_create
       output['vm']['power_state'] == 'off'
     end
 
@@ -50,15 +50,19 @@ module ForemanOpentofu
       raise StandardError, "VM with UUID #{uuid} does not exist" unless tf_state
       vm_command_errors('update vm') do
         attrs = attrs.empty? ? {} : attrs.first
-        data = client({ 'name' => tf_state.name }.merge(attrs)).run('create')
+        data = client({ 'name' => tf_state.name }.merge(attrs)).run_create
         ComputeVM.new(self, data)
       end
     end
 
+    def fetch_resource(resource_name = '', options = {})
+      client({ 'resource' => { name: resource_name, options: options } }).run_fetch
+    end
+
     def test_connection(options = {})
       super
       begin
-        client.run('test_connection')
+        client.run_test_connection
       rescue StandardError => e
         Rails.logger.error("OpenTofu test connection failed: #{e.message}")
         errors.add(:base, e.message)

data/app/models/foreman_opentofu/tofu.rb

@@ -18,6 +18,7 @@
 module ForemanOpentofu
   class Tofu < ComputeResource
     include OpentofuVMCommands
+    include ComputeResourceCaching
     validates :provider, presence: true, inclusion: { in: %w[Tofu] }
     validates :url, presence: true
     validates :user, presence: true
@@ -26,12 +27,19 @@ module ForemanOpentofu
     # alias_attribute :username, :user
     # alias_attribute :endpoint, :url
 
-    delegate :available_attributes, to: :tofu_provider
+    delegate :available_attributes, :capabilities, to: :tofu_provider
+
+    def available_images
+      # make sure available_images can use this CR, e.g. for requesting data_source
+      tofu_provider.available_images(self)
+    end
 
     def provided_attributes
-      super.merge(
-        mac: :mac
-      )
+      super.merge(tofu_provider.provided_attributes || {})
+    end
+
+    def user_data_supported?
+      true
     end
 
     def opentofu_provider
@@ -60,10 +68,6 @@ module ForemanOpentofu
       'OpenTofu'
     end
 
-    def capabilities
-      [:build]
-    end
-
     def self.model_name
       ComputeResource.model_name
     end
@@ -76,6 +80,12 @@ module ForemanOpentofu
       true
     end
 
+    def vm_ready(vm)
+      return tofu_provider.vm_ready(vm) if tofu_provider.respond_to? :vm_ready
+
+      vm.wait_for { ready? }
+    end
+
     def tofu_provider
       ProviderTypeManager.find(opentofu_provider)
     end
@@ -84,8 +94,23 @@ module ForemanOpentofu
       { compute_attributes: {} }
     end
 
+    def new_volume
+      { compute_attributes: {} }
+    end
+
     def editable_network_interfaces?
       true
     end
+
+    def available_resource(resource_name, options = {})
+      cache.cache("#{name}_#{resource_name}") do
+        resource = fetch_resource(resource_name, options)
+        resource.map { |h| OpenStruct.new(h) }
+      end
+    end
+
+    def available_resource_ui_select(resource_name, options = {})
+      available_resource(resource_name, options)&.map { |obj| [obj['name'], obj['id']] }
+    end
   end
 end

data/app/services/foreman_opentofu/app_wrapper.rb

@@ -1,6 +1,7 @@
 module ForemanOpentofu
   class AppWrapper
-    attr_reader :workdir, :planfile, :conffile
+    include HclFormat
+    attr_reader :workdir
 
     # TODO: for future versions
     #   - manage temp-work-dir; problem: no auto-remove after finished :-(
@@ -9,14 +10,33 @@ module ForemanOpentofu
     #   - use JSON-output for easier parsing
     #   - do we need locking or has the object be atomic
 
-    def initialize(workdir)
+    def initialize(workdir, opts = {})
       @workdir = workdir
-      @planfile = File.join(workdir, 'plan.bin')
-      @conffile = File.join(workdir, 'main.tf')
+      @variables = opts[:variables]
     end
 
-    def base_command
-      'tofu'
+    # rubocop:disable Style/SingleLineMethods
+    def planfile()   File.join(workdir, 'plan.bin')     end
+    def conffile()   File.join(workdir, 'main.tf')      end
+    def vardeffile() File.join(workdir, 'variables.tf') end
+
+    def base_command() 'tofu' end
+    # rubocop:enable Style/SingleLineMethods
+
+    # write variables definition file based on @variables into 'variables.tf'
+    def create_variables_file
+      File.open(vardeffile, 'w') do |f|
+        @variables.each_key do |var|
+          data = { 'type' => :string }
+          data['sensitive'] = var.to_s == 'password'
+
+          f << block_to_hcl(['variable', var], data)
+        end
+      end
+    end
+
+    def variable_params
+      @variables.map { |var, val| ['-var', "#{var}=#{val}"] }.flatten
     end
 
     def default_params
@@ -32,6 +52,8 @@ module ForemanOpentofu
     #   end
     # end
     def init(params = [], &block)
+      create_variables_file if @variables
+
       tofu_execute('init', ['-input=false'].concat(parse_params(params)), &block)
     end
 
@@ -78,18 +100,19 @@ module ForemanOpentofu
       cmd.map { |item| "'#{item}'" }.append('2>&1').join(' ')
     end
 
+    def envvars
+      @variables.transform_keys { |variable| "TF_VAR_#{variable}" }
+    end
+
     def execute(cmd)
       output = nil
       # quote cmdline parameters and add stderr to stdout
       commandline = command(cmd)
-      Dir.chdir(workdir) do
-        Rails.logger.debug "Start command: #{commandline.inspect}"
-        IO.popen(commandline, 'r+') do |pipe|
-          if block_given?
-            yield pipe
-          else
-            output = pipe.read
-          end
+      IO.popen(envvars, commandline, 'r+', chdir: workdir) do |pipe|
+        if block_given?
+          yield pipe
+        else
+          output = pipe.read
         end
       end
       ret = $CHILD_STATUS