ConfigLMM 0.1.0 → 0.2.0

data/Plugins/OS/Linux/Linux.lmm.rb

@@ -0,0 +1,256 @@
+
+require 'addressable/uri'
+require 'http'
+require 'securerandom'
+require 'shellwords'
+
+module ConfigLMM
+    module LMM
+        class Linux < Framework::LinuxApp
+
+            ISO_LOCATION = '~/.cache/configlmm/images/'
+            HOSTS_FILE = '/etc/hosts'
+            SSH_CONFIG = '~/.ssh/config'
+            SYSCTL_FILE = '/etc/sysctl.d/10-configlmm.conf'
+
+            def actionLinuxBuild(id, target, activeState, context, options)
+                prepareConfig(target)
+                buildHostsFile(id, target, options)
+                buildSSHConfig(id, target, options)
+                buildAutoYaST(id, target, options)
+            end
+
+            def actionLinuxDeploy(id, target, activeState, context, options)
+                prepareConfig(target)
+                if target['Location'] && target['Location'] != '@me'
+                    uri = Addressable::URI.parse(target['Location'])
+                    case uri.scheme
+                    when 'qemu'
+                        deployOverLibvirt(id, target, activeState, context, options)
+                    when 'ssh'
+                        deployOverSSH(uri, id, target, activeState, context, options)
+                    else
+                        raise Framework::PluginProcessError.new("#{id}: Unknown protocol: #{uri.scheme}!")
+                    end
+                else
+                    deployLocalHostsFile(target, options)
+                    deployLocalSSHConfig(target, options)
+                end
+                if target['AlternativeLocation']
+                    uri = Addressable::URI.parse(target['AlternativeLocation'])
+                    raise Framework::PluginProcessError.new("#{id}: Unsupported protocol: #{uri.scheme}!") if uri.scheme != 'ssh'
+                    deployOverSSH(uri, id, target, activeState, context, options)
+                end
+            end
+
+            def deployOverSSH(locationUri, id, target, activeState, context, options)
+                if target['Domain'] || target['Hosts']
+                    hostsLines = []
+                    if target['Domain']
+                        self.class.sshStart(locationUri) do |ssh|
+                            envs = self.class.sshExec!(ssh, "env").split("\n")
+                            envVars = Hash[envs.map { |vars| vars.split('=', 2) }]
+                            ipAddr = envVars['SSH_CONNECTION'].split[-2]
+                            hostsLines << ipAddr.ljust(16) + target['Domain'] + ' ' + target['Name'] + "\n"
+                        end
+                    end
+                    target['Hosts'].to_a.each do |ip, entries|
+                            hostsLines << ip.ljust(16) + entries.join(' ') + "\n"
+                    end
+                    updateRemoteFile(locationUri, HOSTS_FILE, options, false) do |fileLines|
+                        fileLines + hostsLines
+                    end
+                end
+                if target['Sysctl']
+                    updateRemoteFile(locationUri, SYSCTL_FILE, options, false) do |fileLines|
+                        target['Sysctl'].each do |name, value|
+                            fileLines << "#{name} = #{value}\n"
+                        end
+                        fileLines
+                    end
+                end
+            end
+
+            def deployOverLibvirt(id, target, activeState, context, options)
+                location = Libvirt.getLocation(target['Location'])
+                iso = installationISO(target['Distro'], location)
+                iso = buildISOAutoYaST(id, iso, target, options) if target['Distro'] == SUSE_NAME
+                plugins[:Libvirt].createVM(target['Name'], target, target['Location'], iso, activeState)
+            end
+
+            def buildHostsFile(id, target, options)
+                if target['Hosts']
+                    hosts  = "#\n"
+                    hosts += "# /etc/hosts: static lookup table for host names\n"
+                    hosts += "#\n\n"
+                    hosts += "#<ip-address>   <hostname.domain.org>   <hostname>\n"
+                    hosts += "127.0.0.1       localhost\n"
+                    hosts += "::1             localhost\n\n"
+                    hosts += CONFIGLMM_SECTION_BEGIN
+                    target['Hosts'].each do |ip, entries|
+                        hosts += ip.ljust(16) + entries.join(' ') + "\n"
+                    end
+                    hosts += CONFIGLMM_SECTION_END
+
+                    path = options['output'] + '/' + id
+                    mkdir(path + '/etc', options[:dry])
+                    fileWrite(path + HOSTS_FILE, hosts, options[:dry])
+                end
+            end
+
+            def buildSSHConfig(id, target, options)
+                if !target['SSH']['Config'].empty?
+                    sshConfig  = "\n"
+                    sshConfig += CONFIGLMM_SECTION_BEGIN
+                    target['SSH']['Config'].each do |name, info|
+                        sshConfig += "Host #{name} #{info['HostName']}\n"
+                        sshConfig += "    HostName " + info['HostName'] + "\n" if info['HostName']
+                        sshConfig += "    Port " + info['Port'] + "\n" if info['Port']
+                        sshConfig += "    User " + info['User'] + "\n" if info['User']
+                        sshConfig += "    IdentityFile " + info['IdentityFile'] + "\n" if info['IdentityFile']
+                        sshConfig += "\n"
+                    end
+                    sshConfig += CONFIGLMM_SECTION_END
+                    sshConfig += "\n"
+
+                    configPath = options['output'] + '/' + id
+                    mkdir(configPath + '/root/.ssh', options[:dry])
+                    fileWrite(configPath + SSH_CONFIG.gsub('~', '/root'), sshConfig, options[:dry])
+                end
+            end
+
+            def buildAutoYaST(id, target, options)
+                if target['Distro'] == SUSE_NAME
+                    outputFolder = options['output'] + '/' + id + '/'
+                    template = ERB.new(File.read(__dir__ + '/openSUSE/autoinst.xml.erb'))
+                    renderTemplate(template, target, outputFolder + 'autoinst.xml', options)
+                end
+            end
+
+            def deployLocalHostsFile(target, options)
+                if target['Hosts']
+                    updateLocalFile(HOSTS_FILE, options) do |hostsLines|
+                        target['Hosts'].each do |ip, entries|
+                            hostsLines << ip.ljust(16) + entries.join(' ') + "\n"
+                        end
+                        hostsLines
+                    end
+                end
+            end
+
+            def deployLocalSSHConfig(target, options)
+                if !target['SSH']['Config'].empty?
+                    updateLocalFile(File.expand_path(SSH_CONFIG), options) do |configLines|
+                        target['SSH']['Config'].each do |name, info|
+                            configLines << "Host #{name} #{info['HostName']}\n"
+                            configLines << "    HostName " + info['HostName'] + "\n" if info['HostName']
+                            configLines << "    Port " + info['Port'] + "\n" if info['Port']
+                            configLines << "    User " + info['User'] + "\n" if info['User']
+                            configLines << "    IdentityFile " + info['IdentityFile'] + "\n" if info['IdentityFile']
+                        end
+                        configLines
+                    end
+                end
+            end
+
+            def installationISO(distro, location)
+                url = nil
+                case distro
+                when SUSE_NAME
+                    if location.empty?
+                        # TODO automatically fetch latest version from website
+                        url = 'https://download.opensuse.org/distribution/leap/15.6/iso/openSUSE-Leap-15.6-NET-x86_64-Media.iso'
+                    else
+                        raise Framework::PluginProcessError.new("#{id}: Unimplemented!")
+                    end
+                else
+                    raise Framework::PluginProcessError.new("#{id}: Unknown Linux Distro: #{distro}!")
+                end
+
+                filename = File.basename(Addressable::URI.parse(url).path)
+                iso = File.expand_path(ISO_LOCATION + filename)
+                if !File.exist?(iso)
+                    mkdir(File.expand_path(ISO_LOCATION), false)
+                    prompt.say('Downloading... ' + url)
+                    response = HTTP.follow.get(url)
+                    raise "Failed to download file: #{response.status}" unless response.status.success?
+                    File.open(iso, 'wb') do |file|
+                        response.body.each do |chunk|
+                            file.write(chunk)
+                        end
+                    end
+                end
+                iso
+            end
+
+            def buildISOAutoYaST(id, iso, target, options)
+                outputFolder = options['output'] + '/iso/'
+                mkdir(outputFolder, false)
+                `xorriso -osirrox on -indev #{iso} -extract / #{outputFolder} 2>&1 >/dev/null`
+                FileUtils.chmod_R(0750, outputFolder) # Need to make it writeable so it can be deleted
+                copy(options['output'] + '/' + id + '/autoinst.xml', outputFolder, false)
+
+                cfg = outputFolder + "boot/x86_64/loader/isolinux.cfg"
+                `sed -i 's|default harddisk|default linux|' #{cfg}`
+                `sed -i 's|append initrd=initrd splash=silent showopts|append initrd=initrd splash=silent autoyast=device://sr0/autoinst.xml|' #{cfg}`
+                `sed -i 's|prompt		1|prompt		0|' #{cfg}`
+                `sed -i 's|timeout		600|timeout		1|' #{cfg}`
+
+                patchedIso = File.dirname(iso) + '/patched.iso'
+                `xorriso -as mkisofs -no-emul-boot -boot-load-size 4 -boot-info-table -iso-level 4 -b boot/x86_64/loader/isolinux.bin -c boot/x86_64/loader/boot.cat -eltorito-alt-boot -e boot/x86_64/efi -no-emul-boot -o #{patchedIso} #{outputFolder} 2>&1 >/dev/null`
+                patchedIso
+            end
+
+            def prepareConfig(target)
+                target['SSH'] ||= {}
+                target['SSH']['Config'] ||= {}
+                target['Users'] ||= {}
+                target['HostName'] = target['Name'] unless target['HostName']
+
+                if ENV['LINUX_ROOT_PASSWORD_HASH']
+                    target['Users']['root'] ||= {}
+                    target['Users']['root']['PasswordHash'] = ENV['LINUX_ROOT_PASSWORD_HASH']
+                elsif ENV['LINUX_ROOT_PASSWORD']
+                    target['Users']['root'] ||= {}
+                    target['Users']['root']['PasswordHash'] = self.class.linuxPasswordHash(ENV['LINUX_ROOT_PASSWORD'])
+                elsif target['Users'].key?('root')
+                    if !target['Users']['root']['Password'] &&
+                       !target['Users']['root']['PasswordHash']
+                        rootPassword = SecureRandom.urlsafe_base64(12)
+                        prompt.say("Root password: #{rootPassword}", :color => :magenta)
+                        target['Users']['root']['PasswordHash'] = self.class.linuxPasswordHash(rootPassword)
+                    elsif target['Users']['root']['Password'] == 'no'
+                        target['Users']['root'].delete('Password')
+                    end
+                end
+
+                target['Users'].each do |user, info|
+                    newKeys = []
+                    info['AuthorizedKeys'].to_a.each do |key|
+                        if key.start_with?('/') || key.start_with?('~')
+                            newKeys << File.read(File.expand_path(key)).strip
+                        else
+                            newKeys << key
+                        end
+                    end
+                    info['AuthorizedKeys'] = newKeys
+                end
+
+                packages = YAML.load_file(__dir__ + '/Packages.yaml')
+                newApps = []
+                target['Services'] ||= []
+                if target['Apps'].to_a.include?('sshd')
+                    target['Services'] << 'sshd'
+                    target['Services'].uniq!
+                end
+                target['Apps'] = self.class.mapPackages(target['Apps'], target['Distro'])
+            end
+
+            def self.linuxPasswordHash(password)
+                salt = SecureRandom.alphanumeric(16)
+                password.crypt('$6$' + salt)
+            end
+
+        end
+    end
+end

data/Plugins/OS/Linux/Packages.yaml

@@ -0,0 +1,13 @@
+Arch Linux:
+    sshd: openssh
+    Postfix: postfix
+    Dovecot: dovecot
+    PostgreSQL: postgresql
+    Valkey: redis
+
+openSUSE Leap:
+    sshd: openssh
+    Postfix: postfix
+    Dovecot: dovecot
+    PostgreSQL: postgresql-server
+    Valkey: redis

data/Plugins/OS/Linux/openSUSE/autoinst.xml.erb

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE profile>
+<profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns">
+  <general config:type="map">
+    <mode config:type="map">
+      <confirm config:type="boolean">false</confirm>
+    </mode>
+  </general>
+  <bootloader t="map">
+    <global t="map">
+      <append>splash=silent preempt=full mitigations=auto quiet security=apparmor console=ttyS0,115200</append>
+    </global>
+  </bootloader>
+  <host t="map">
+    <% if !config['Hosts'].to_a.empty? %>
+      <hosts t="list">
+        <% config['Hosts'].each do |address, names| %>
+          <hosts_entry t="map">
+            <host_address><%= address %></host_address>
+            <names t="list">
+              <% names.each do |name| %>
+                <name><%= name %></name>
+              <% end %>
+            </names>
+          </hosts_entry>
+        <% end %>
+      </hosts>
+    <% end %>
+  </host>
+  <networking t="map">
+    <dns t="map">
+      <hostname><%= config['HostName'] %></hostname>
+      <% if config['Domain'] %>
+        <domain><%= config['Domain'] %></domain>
+      <% end %>
+    </dns>
+  </networking>
+  <software t="map">
+    <% if !config['Apps'].to_a.empty? %>
+      <packages t="list">
+        <% config['Apps'].each do |app| %>
+          <package><%= app.downcase %></package>
+        <% end %>
+      </packages>
+    <% end %>
+  </software>
+  <services-manager t="map">
+    <services t="map">
+      <% if !config['Services'].to_a.empty? %>
+        <enable t="list">
+          <% config['Services'].each do |service| %>
+            <service><%= service %></service>
+          <% end %>
+        </enable>
+      <% end %>
+    </services>
+  </services-manager>
+  <timezone t="map">
+    <timezone>Etc/UTC</timezone>
+  </timezone>
+  <% if !config['Users'].to_h.empty? %>
+    <users config:type="list">
+      <% config['Users'].each do |user, info| %>
+        <user>
+          <username>root</username>
+          <% if info['PasswordHash'] %>
+            <encrypted config:type="boolean">true</encrypted>
+            <user_password><%= info['PasswordHash'] %></user_password>
+          <% elsif info['Password'] %>
+            <encrypted config:type="boolean">false</encrypted>
+            <user_password><%= info['Password'] %></user_password>
+          <% end %>
+          <% if info['Shell'] %>
+            <shell>/usr/bin/<%= info['Shell'] %></shell>
+          <% end %>
+          <% if !info['AuthorizedKeys'].to_a.empty? %>
+            <authorized_keys config:type="list">
+              <% info['AuthorizedKeys'].each do |entry| %>
+                <listentry><%= entry %></listentry>
+              <% end %>
+            </authorized_keys>
+          <% end %>
+        </user>
+      <% end %>
+    </users>
+  <% end %>
+</profile>

data/Plugins/Platforms/GoDaddy/GoDaddy.lmm.rb

@@ -26,7 +26,8 @@ module ConfigLMM
                     data.each do |name, data|
                         self.processDNS(domain, data).each do |type, records|
                             records.each do |record|
-                                shortName = name
+                                shortName = Addressable::IDNA.to_ascii(name)
+                                record[:content] = Addressable::IDNA.to_ascii(record[:content]) if record[:type] == 'CNAME'
                                 config['Records'] += [shortName, record[:ttl], ' IN ', record[:type], record[:content]].join("\t") + "\n"
                             end
                         end

data/Plugins/Platforms/libvirt/libvirt.lmm.rb

@@ -0,0 +1,103 @@
+
+require 'fog/libvirt'
+require 'filesize'
+
+module ConfigLMM
+    module LMM
+        class Libvirt < Framework::Plugin
+
+            DEFAULT_IMAGE_PATH = '~/.local/share/libvirt/images/'
+            DEFAULT_VRAM = 16 * 1024 # 16 MiB
+
+            def actionLibvirtDeploy(id, target, activeState, context, options)
+                if !target['Location']
+                    target['Location'] = 'qemu:///session'
+                end
+                compute = Fog::Compute.new(provider: :libvirt, libvirt_uri: target['Location'])
+                createPools(target, compute, self.class.isLocal?(target['Location']))
+            end
+
+            def createPools(target, compute, isLocal)
+                if target['Pools']
+                    allPools = compute.pools.all
+                    target['Pools'].each do |name, location|
+                        next if allPools.find { |pool| pool.name == name }
+                        location = DEFAULT_IMAGE_PATH unless location
+                        location = File.expand_path(location) if isLocal
+                        xml = dirPoolXML(name, location)
+                        pool = compute.pools.create(persistent: true, autostart: true, xml: xml)
+                        pool.build
+                        pool.start
+                    end
+                end
+            end
+
+            def createVM(serverName, serverInfo, targetUri, iso, activeState)
+                compute = Fog::Compute.new(provider: :libvirt, libvirt_uri: targetUri)
+                server = compute.servers.all.find { |server| server.name == serverName }
+                if server
+                    server.start
+                    return
+                end
+                settings = {
+                    name: serverName,
+                    cpu: {
+                        mode: 'host-passthrough'
+                    },
+                    video: { type: 'qxl', vram: DEFAULT_VRAM }
+                }
+                if serverInfo['CPU']
+                    settings[:cpus] = serverInfo['CPU']
+                end
+                if serverInfo['RAM']
+                    settings[:memory_size] = Filesize.from(serverInfo['RAM']).to_f('KiB').to_i
+                end
+                volumeName = serverName + '.img'
+                volume = compute.volumes.all.find { |volume| volume.name == volumeName }
+                if volume
+                    settings[:volumes] = [volume]
+                elsif serverInfo['Storage']
+                    storage = Filesize.from(serverInfo['Storage']).to_f('GiB').to_i
+                    volume = compute.volumes.create(
+                        name: volumeName,
+                        pool_name: compute.pools.first.name,
+                        capacity: storage
+                    )
+                    settings[:volumes] = [volume]
+                end
+                if serverInfo['NetworkBridge']
+                    nic = {
+                        bridge: serverInfo['NetworkBridge'],
+                    }
+                    settings[:nics] = [nic]
+                end
+                server = compute.servers.new(**settings)
+                if iso
+                    server.iso_dir = File.dirname(iso)
+                    server.iso_file = File.basename(iso)
+                end
+                server.save
+                activeState['Status'] = 'CREATED'
+                state.save
+                server.start
+            end
+
+            def dirPoolXML(name, path)
+                xml  = '<pool type="dir">'
+                xml += "    <name>#{name.encode(:xml => :text)}</name>"
+                xml += "<target><path>#{path.encode(:xml => :text)}</path></target>"
+                xml += '</pool>'
+                xml
+            end
+
+            def self.isLocal?(location)
+                self.getLocation(location).empty?
+            end
+
+            def self.getLocation(location)
+                uri = Addressable::URI.parse(location)
+                uri.hostname
+            end
+        end
+    end
+end

data/README.md

@@ -1,5 +1,7 @@
 # ConfigLMM - Large Configuration Management Manager
 
+![Yo Dawg I Heard you like config so I put a config in your Config](/Images/configINconfig.png)
+
 ## Manage The Management with ease!
 
 You define how you want your applications/systems/containers/services/servers
@@ -57,6 +59,8 @@ to work without being vendor locked into any particular implementation or provid
 
 The true [GitOps](https://en.wikipedia.org/wiki/DevOps#GitOps)/DevOps/DevSecOps/[TestOps](https://en.wikipedia.org/wiki/TestOps)/SysOps/[AIOps](https://en.wikipedia.org/wiki/Artificial_Intelligence_for_IT_Operations)[DataOps](https://en.wikipedia.org/wiki/DataOps) which I call AllTheOps :)
 
+![One Config to Rule Them All](/Images/singleConfig.png)
+
 ## Benefits
 
 * Compare performance and price among different providers

data/lib/ConfigLMM/Framework/plugins/linuxApp.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module ConfigLMM
+    module Framework
+
+        class LinuxApp < Framework::Plugin
+
+            LINUX_FOLDER = __dir__ + '/../../../../Plugins/OS/Linux/'
+            SUSE_NAME = 'openSUSE Leap'
+            SUSE_ID = 'opensuse-leap'
+
+            def ensurePackage(name, location)
+              self.class.ensurePackage(name, location)
+            end
+
+            def self.ensurePackage(name, location)
+                if location && location != '@me'
+                    uri = Addressable::URI.parse(location)
+                    raise Framework::PluginProcessError.new("#{id}: Unknown Protocol: #{uri.scheme}!") if uri.scheme != 'ssh'
+                    self.ensurePackageOverSSH(name, uri)
+                else
+                    # TODO
+                end
+            end
+
+            def self.ensurePackageOverSSH(name, locationOrSSH)
+
+                closure = Proc.new do |ssh|
+                    distroInfo = self.distroInfoFromSSH(ssh)
+                    pkgs = self.mapPackages([name], distroInfo['Name']).first
+
+                    pkgs = [pkgs] unless pkgs.is_a?(Array)
+                    command = distroInfo['InstallPackage'] + ' ' + pkgs.map { |pkg| pkg.shellescape }.join(' ')
+                    self.sshExec!(ssh, command)
+                end
+
+                if locationOrSSH.is_a?(String) || locationOrSSH.is_a?(Addressable::URI)
+                    self.sshStart(locationOrSSH) do |ssh|
+                        closure.call(ssh)
+                    end
+                else
+                  closure.call(locationOrSSH)
+                end
+
+            end
+
+            def ensureServiceAutoStart(name, location)
+                if location && location != '@me'
+                    uri = Addressable::URI.parse(location)
+                    raise Framework::PluginProcessError.new("#{id}: Unknown Protocol: #{uri.scheme}!") if uri.scheme != 'ssh'
+                    self.class.sshStart(uri) do |ssh|
+                        distroInfo = self.class.distroInfoFromSSH(ssh)
+
+                        command = distroInfo['AutoStartService'] + ' ' + name.shellescape
+                        self.class.sshExec!(ssh, command)
+                    end
+                else
+                    # TODO
+                end
+            end
+
+            def startService(name, location)
+                if location && location != '@me'
+                    uri = Addressable::URI.parse(location)
+                    raise Framework::PluginProcessError.new("#{id}: Unknown Protocol: #{uri.scheme}!") if uri.scheme != 'ssh'
+                    self.class.sshStart(uri) do |ssh|
+                        distroInfo = self.class.distroInfoFromSSH(ssh)
+
+                        command = distroInfo['StartService'] + ' ' + name.shellescape
+                        self.class.sshExec!(ssh, command)
+                    end
+                else
+                    # TODO
+                end
+            end
+
+            def self.mapPackages(packages, distroName)
+                distroPackages = YAML.load_file(LINUX_FOLDER + 'Packages.yaml')
+                names = []
+                packages.to_a.each do |pkg|
+                    packageName = distroPackages[distroName][pkg]
+                    if packageName
+                        names << packageName
+                    else
+                        names << pkg.downcase
+                    end
+                end
+                names
+            end
+
+            def self.createSubuidsOverSSH(user, distroInfo, ssh)
+                self.sshExec!(ssh, "#{distroInfo['ModifyUser']} --add-subuids 100000-165535 --add-subgids 100000-165535 #{user}")
+            end
+
+            def self.distroID
+                `cat /etc/os-release | grep "^ID=" | cut -d "=" -f 2`.strip.gsub('"', '')
+            end
+
+            def self.distroIDfromSSH(ssh)
+                ssh.exec!('cat /etc/os-release | grep "^ID=" | cut -d "=" -f 2').strip.gsub('"', '')
+            end
+
+            def self.distroInfoFromSSH(ssh)
+                distroInfo = self.distroInfo(self.distroIDfromSSH(ssh))
+            end
+
+            def self.distroInfo(distroID)
+                distributions = YAML.load_file(LINUX_FOLDER + 'Distributions.yaml')
+                raise Framework::PluginProcessError.new("Unknown Linux Distro: #{distroID}!") unless distributions.key?(distroID)
+                distributions[distroID]
+            end
+
+        end
+    end
+end