ConfigLMM 0.3.0 → 0.5.0

data/Plugins/Apps/ClickHouse/ClickHouse.container

@@ -0,0 +1,28 @@
+
+[Unit]
+Description=ClickHouse container
+After=local-fs.target
+
+[Container]
+ContainerName=ClickHouse
+Image=docker.io/clickhouse/clickhouse-server:latest
+EnvironmentFile=/var/lib/clickhouse/.config/containers/systemd/ClickHouse.env
+Network=slirp4netns:allow_host_loopback=true
+PublishPort=0.0.0.0:8123:8123
+PublishPort=0.0.0.0:19100:9000
+UserNS=keep-id:uid=1000,gid=1000
+Volume=/var/lib/clickhouse/server/config.d:/etc/clickhouse-server/config.d
+Volume=/var/lib/clickhouse/server/users.d:/etc/clickhouse-server/users.d
+Volume=/var/lib/clickhouse/data:/var/lib/clickhouse
+Volume=/var/lib/clickhouse/logs:/var/log/clickhouse-server
+LogDriver=journald
+AddCapability=IPC_LOCK
+AddCapability=SYS_NICE
+AutoUpdate=registry
+
+[Service]
+TimeoutStartSec=6min
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target default.target

data/Plugins/Apps/ClickHouse/ClickHouse.lmm.rb

@@ -0,0 +1,113 @@
+
+require_relative 'Connection'
+
+module ConfigLMM
+    module LMM
+        class ClickHouse < Framework::Plugin
+
+            USER = 'clickhouse'
+            HOME_DIR = '/var/lib/clickhouse'
+            SERVICE_NAME = 'ClickHouse'
+            CONTAINER_NAME = 'ClickHouse'
+            PORT = '19100'
+            DEFAULT_CLUSTER = 'default'
+
+            def actionClickHouseDeploy(id, target, activeState, context, options)
+                self.withConnection(target['Location'], target) do |connection|
+                    Linux.withConnection(connection) do |linuxConnection|
+                        Podman.ensurePresent(linuxConnection, options)
+                        Podman.createUser(USER, HOME_DIR, 'ClickHouse', linuxConnection, options)
+                        linuxConnection.withUserShell(USER) do |shell|
+                            shell.createDirs(options, '~/data', '~/logs', '~/server/config.d' ,'~/server/users.d')
+                        end
+
+                        user = target['AdminUsername'] || context.secrets.load(target['SecretId'], 'USERNAME') || 'admin'
+                        context.secrets.store(target['SecretId'], 'USERNAME', user)
+                        password = context.secrets.load(target['SecretId'], 'PASSWORD')
+                        if password.nil?
+                            password = SecureRandom.alphanumeric(20)
+                            context.secrets.store(target['SecretId'], 'PASSWORD', password)
+                            context.secrets.print("#{user} password", password)
+                        end
+
+                        path = Podman.containersPath(HOME_DIR)
+                        linuxConnection.fileWrite("#{path}/ClickHouse.env", "CLICKHOUSE_USER=#{user}", options)
+                        linuxConnection.fileAppend("#{path}/ClickHouse.env", "CLICKHOUSE_PASSWORD=#{password}", { **options, hide: true })
+                        linuxConnection.fileAppend("#{path}/ClickHouse.env", "CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1", options)
+
+                        linuxConnection.setUserGroup("#{path}/ClickHouse.env", USER, USER, options)
+                        linuxConnection.setPrivate("#{path}/ClickHouse.env", options)
+
+                        loggerConfigFile = __dir__ + '/Config/logger.yaml'
+                        if target['LogLevel']
+                            loggerConfig = YAML.load_file(loggerConfigFile)
+                            loggerConfig['logger']['level'] = target['LogLevel']
+                            loggerConfigFile = options['output'] + '/logger.yaml'
+                            File.write(loggerConfigFile, loggerConfig.to_yaml)
+                        end
+
+                        linuxConnection.upload(__dir__ + '/ClickHouse.container', path, options)
+                        linuxConnection.upload(__dir__ + '/Config/listen.yaml', HOME_DIR + '/server/config.d/', options)
+                        linuxConnection.upload(loggerConfigFile, HOME_DIR + '/server/config.d/', options)
+                        linuxConnection.upload(__dir__ + '/Config/zookeepers.yaml', HOME_DIR + '/server/config.d/', options)
+                        linuxConnection.reloadUserServices(USER, options)
+                        linuxConnection.restartUserService(USER, SERVICE_NAME, options)
+                    end
+                end
+            end
+
+            def self.defaults(settings)
+                settings['HostName'] = 'localhost' unless settings['HostName']
+                settings['Port'] = PORT unless settings['Port']
+            end
+
+            def self.withConnection(settings, linuxConnection, secrets, options)
+                if settings['HostName'].nil? || settings['HostName'] == 'localhost'
+                    settings = settings.dup
+                    settings.delete('HostName')
+                    settings.delete('Port')
+                    linuxConnection.withUserShell(USER) do |shellConnection|
+                        Podman.withConnection(shellConnection, Podman.container(CONTAINER_NAME, shellConnection, options)) do |connection|
+                            yield(ClickHouseConnection.new(connection, settings))
+                        end
+                    end
+                else
+                    if settings['ClickHouseSecretId'].nil?
+                        raise Framework::PluginError.new('You need to set ClickHouseSecretId!')
+                    end
+
+                    self.defaults(settings)
+                    adminUsername = secrets.load(settings['ClickHouseSecretId'], 'USERNAME')
+                    adminPassword = secrets.load(settings['ClickHouseSecretId'], 'PASSWORD')
+
+                    if adminUsername.nil? || adminPassword.nil?
+                        raise Framework::PluginError.new('Invalid ClickHouseSecretId!')
+                    end
+
+                    raise 'Not Implemented!'
+                    yield(nil)
+                end
+            end
+
+            def cleanup(configs, state, context, options)
+                cleanupType(:ClickHouse, configs, state, context, options) do |item, id, state, context, options, connection|
+                    Linux.withConnection(connection) do |linuxConnection|
+                        linuxConnection.stopUserService(USER, SERVICE_NAME, options)
+
+                        path = Podman.containersPath(HOME_DIR)
+                        linuxConnection.rm("#{path}/ClickHouse.container", options[:dry])
+
+                        state.item(id)['Status'] = State::STATUS_DELETED unless options[:dry]
+
+                        if options[:destroy]
+                            linuxConnection.deleteUserAndGroup(USER, options)
+                            state.item(id)['Status'] = State::STATUS_DESTROYED unless options[:dry]
+                        end
+                    end
+                end
+            end
+
+        end
+    end
+end
+

data/Plugins/Apps/ClickHouse/Config/listen.yaml

@@ -0,0 +1,2 @@
+
+listen_host: 0.0.0.0

data/Plugins/Apps/ClickHouse/Config/logger.yaml

@@ -0,0 +1,8 @@
+
+logger:
+    log:
+    errorlog:
+    console: true
+    level: information
+    formatting:
+        type: json

data/Plugins/Apps/ClickHouse/Config/zookeepers.yaml

@@ -0,0 +1,5 @@
+
+zookeeper:
+    - node:
+          host: 10.0.2.2
+          port: 2181

data/Plugins/Apps/ClickHouse/Connection.rb

@@ -0,0 +1,96 @@
+
+module ConfigLMM
+    module LMM
+        class ClickHouseConnection
+
+            attr_reader :connection
+            attr_reader :settings
+
+            def initialize(connection, settings)
+                @connection = connection
+                @settings = settings
+            end
+
+            def exec(query, allowFailure = false, options = {})
+                cmd = "clickhouse-client --query '#{LinuxShell.escapeSingleQuotes(query)}'"
+                begin
+                    connection.exec(cmd, allowFailure, options)
+                rescue IO::ExecError => error
+                    # Retry
+                    # Code: 210. DB::NetException: Connection refused (localhost:9000). (NETWORK_ERROR)
+                    if error.stderr.include?('NETWORK_ERROR')
+                        sleep(5)
+                        connection.exec(cmd, allowFailure, options)
+                    else
+                        raise
+                    end
+                end
+            end
+
+            def createUser(user, password, cluster, options)
+                clusterSQL = ''
+                if cluster
+                    clusterSQL = " ON CLUSTER #{cluster}"
+                end
+                exec("CREATE USER IF NOT EXISTS #{user}#{clusterSQL}", false, options)
+                if password
+                    sql = "ALTER USER #{user} IDENTIFIED WITH bcrypt_password by '#{password}'"
+                    exec(sql, false, { **options, hide: true })
+                end
+            end
+
+            def dropUser(user, cluster, options = {})
+                clusterSQL = ''
+                if cluster
+                    clusterSQL = " ON CLUSTER #{cluster}"
+                end
+                exec("DROP USER IF EXISTS #{user}#{clusterSQL}", false, options)
+            end
+
+            def createDB(database, cluster, options = {})
+                clusterSQL = ''
+                if cluster
+                    clusterSQL = " ON CLUSTER #{cluster}"
+                end
+                exec("CREATE DATABASE IF NOT EXISTS #{database}#{clusterSQL}", false, options)
+            end
+
+            def dropDB(database, cluster, options = {})
+                clusterSQL = ''
+                if cluster
+                    clusterSQL = " ON CLUSTER #{cluster}"
+                end
+                exec("DROP DATABASE IF EXISTS #{database}#{clusterSQL}", false, options)
+            end
+
+            def grant(privilege, user, target, cluster, options = {})
+                clusterSQL = ''
+                if cluster
+                    clusterSQL = "ON CLUSTER #{cluster}"
+                end
+                exec("GRANT #{clusterSQL} #{privilege} ON #{target} TO #{user}", false, options)
+            end
+
+            def grantDB(privilege, user, database, cluster, options = {})
+                grant(privilege, user, database + '.*', cluster, options)
+            end
+
+            def grantCluster(user, cluster, options = {})
+                clusterSQL = ''
+                if cluster
+                    clusterSQL = "ON CLUSTER #{cluster}"
+                end
+                exec("GRANT #{clusterSQL} CLUSTER ON *.* TO #{user}", false, options)
+            end
+
+            def grantRemote(user, cluster, options = {})
+                clusterSQL = ''
+                if cluster
+                    clusterSQL = "ON CLUSTER #{cluster}"
+                end
+                exec("GRANT #{clusterSQL} REMOTE ON *.* TO #{user}", false, options)
+            end
+
+        end
+    end
+end

data/Plugins/Apps/Discourse/Discourse-Sidekiq.container

@@ -0,0 +1,22 @@
+
+
+[Unit]
+Description=Discourse Sidekiq container
+After=local-fs.target
+
+[Container]
+ContainerName=Discourse-Sidekiq
+Image=docker.io/bitnami/discourse:latest
+Exec=/opt/bitnami/scripts/discourse-sidekiq/run.sh
+EnvironmentFile=/var/lib/discourse/.config/containers/systemd/Discourse.env
+Network=slirp4netns:allow_host_loopback=true
+UserNS=keep-id:uid=999,gid=999
+Volume=/var/lib/discourse/sidekiq:/bitnami/discourse
+LogDriver=journald
+AutoUpdate=registry
+
+[Service]
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target default.target

data/Plugins/Apps/Discourse/Discourse.conf.erb

@@ -0,0 +1,38 @@
+
+server {
+
+    <% if !config['TLS'] %>
+        listen       <%= config['Port'] %>;
+        listen  [::]:<%= config['Port'] %>;
+    <% else %>
+        <% if config['NginxVersion'] >= 1.25 %>
+            listen <%= config['Port'] %> ssl;
+            listen [::]:<%= config['Port'] %> ssl;
+            http2 on;
+            http3 on;
+            quic_retry on;
+            add_header Alt-Svc 'h3=":<%= config['Port'] %>"; ma=86400';
+        <% else %>
+            listen <%= config['Port'] %> ssl http2;
+            listen [::]:<%= config['Port'] %> ssl http2;
+        <% end %>
+
+        include config-lmm/ssl.conf;
+    <% end %>
+
+    server_name <%= config['Domain'] %>;
+
+    include config-lmm/gateway-errors.conf;
+    include config-lmm/security.conf;
+
+    location / {
+        <% if config['Server'] %>
+            proxy_pass <%= config['Server'] %>;
+        <% else %>
+            proxy_pass http://127.0.0.1:13100;
+        <% end %>
+
+        include config-lmm/proxy.conf;
+    }
+
+}

data/Plugins/Apps/Discourse/Discourse.container

@@ -0,0 +1,21 @@
+
+[Unit]
+Description=Discourse container
+After=local-fs.target
+
+[Container]
+ContainerName=Discourse
+Image=docker.io/bitnami/discourse:latest
+EnvironmentFile=/var/lib/discourse/.config/containers/systemd/Discourse.env
+Network=slirp4netns:allow_host_loopback=true
+PublishPort=127.0.0.1:13100:3000
+UserNS=keep-id:uid=999,gid=999
+Volume=/var/lib/discourse/data:/bitnami/discourse
+LogDriver=journald
+AutoUpdate=registry
+
+[Service]
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target default.target

data/Plugins/Apps/Discourse/Discourse.lmm.rb

@@ -0,0 +1,156 @@
+
+module ConfigLMM
+    module LMM
+        class Discourse < Framework::Plugin
+
+            USER = 'discourse'
+            HOME_DIR = '/var/lib/discourse'
+            HOST_IP = '10.0.2.2'
+            CONTAINER_NAME = 'Discourse'
+
+            def actionDiscourseDeploy(id, target, activeState, context, options)
+                raise Framework::PluginProcessError.new('Domain field must be set!') unless target['Domain']
+
+                target['Database'] ||= {}
+                self.withConnection(target['Location'], target) do |connection|
+                    Linux.withConnection(connection) do |linuxConnection|
+                        target['Database'] ||= {}
+
+                        dbPassword = self.configurePostgreSQL(target['Database'], linuxConnection, options)
+
+                        Podman.ensurePresent(linuxConnection, options)
+                        Podman.createUser(USER, HOME_DIR, 'Discourse', linuxConnection, options)
+                        linuxConnection.withUserShell(USER) do |shell|
+                            shell.createDirs(options, '~/data', '~/sidekiq')
+                        end
+
+                        path = Podman.containersPath(HOME_DIR)
+                        linuxConnection.fileWrite("#{path}/Discourse.env", "DISCOURSE_DATABASE_HOST=#{HOST_IP}", options)
+                        linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_DATABASE_NAME=#{USER}", options)
+                        linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_DATABASE_USER=#{USER}", options)
+                        linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_DATABASE_PASSWORD=#{dbPassword}", { **options, hide: true })
+                        linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_HOST=#{target['Domain']}", options)
+                        linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_REDIS_HOST=#{HOST_IP}", options)
+
+                        if target['ValkeySecretId']
+                            linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_REDIS_PASSWORD=#{context.secrets.load(target['ValkeySecretId'], 'VALKEY_PASSWORD')}", { **options, hide: true })
+                        end
+
+                        username = 'user'
+                        if target['Admin']
+                            if target['Admin']['Username']
+                                linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_USERNAME=#{target['Admin']['Username']}", options)
+                            end
+
+                            if target['Admin']['EMail']
+                                linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_EMAIL=#{target['Admin']['EMail']}", options)
+                            end
+
+                            if target['Admin'].key?('FirstName')
+                                firstName = target['Admin']['FirstName'].to_s
+                                firstName = '""' if firstName.empty?
+                                linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_FIRST_NAME=#{firstName}", options)
+                            end
+
+                            if target['Admin'].key?('LastName')
+                                lastName = target['Admin']['LastName'].to_s
+                                lastName = '""' if lastName.empty?
+                                linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_LAST_NAME=#{lastName}", options)
+                            end
+                        end
+
+                        adminPassword = context.secrets.load(target['SecretId'], 'ADMIN_PASSWORD')
+                        if adminPassword.nil?
+                            adminPassword = SecureRandom.alphanumeric(20)
+                            context.secrets.store(target['SecretId'], 'ADMIN_PASSWORD', adminPassword)
+                            context.secrets.print("Discourse Admin '#{username}' password", adminPassword)
+                        end
+                        linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_PASSWORD=#{adminPassword}", { **options, hide: true })
+
+                        if target['SMTP']
+                            host = target['SMTP']['Host']
+                            host = HOST_IP if host.to_s.empty? || ['localhost', '127.0.0.1'].include?(host)
+
+                            linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_SMTP_HOST=#{host}", options)
+                            linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_SMTP_PORT_NUMBER=#{target['SMTP']['Port']}", options)
+                            linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_SMTP_USER=#{target['SMTP']['Username']}", options)
+
+                            smtpPassword = ''
+                            if target['SMTP']['SecretId']
+                                smtpPassword = context.secrets.load(target['SMTP']['SecretId'], target['SMTP']['Username'].upcase + '_PASSWORD')
+                            end
+                            linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_SMTP_PASSWORD=#{smtpPassword}", { **options, hide: true })
+
+                            auth = target['SMTP']['Auth'].to_s.downcase
+                            auth = 'plain' if auth.empty?
+                            linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_SMTP_AUTH=#{auth}", options)
+                            if target['SMTP']['Port'] == 465
+                                linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_EXTRA_CONF_CONTENT=smtp_force_tls = true", options)
+                            end
+                        else
+                            linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_SMTP_HOST=#{HOST_IP}", options)
+                            linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_SMTP_PORT_NUMBER=25", options)
+                            linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_SMTP_USER=\"\"", options)
+                            linuxConnection.fileAppend("#{path}/Discourse.env", "DISCOURSE_SMTP_PASSWORD=\"\"", options)
+                        end
+
+                        linuxConnection.fileAppend("#{path}/Discourse.env", 'DISCOURSE_PRECOMPILE_ASSETS=no', options)
+                        linuxConnection.fileAppend("#{path}/Discourse.env", 'CHEAP_SOURCE_MAPS=1', options)
+                        linuxConnection.fileAppend("#{path}/Discourse.env", 'JOBS=1', options)
+
+                        linuxConnection.setUserGroup("#{path}/Discourse.env", USER, USER, options)
+                        linuxConnection.setPrivate("#{path}/Discourse.env", options)
+
+                        linuxConnection.upload(__dir__ + '/Discourse.container', path, options)
+                        linuxConnection.upload(__dir__ + '/Discourse-Sidekiq.container', path, options)
+
+                        linuxConnection.reloadUserServices(USER, options)
+                        linuxConnection.restartUserService(USER, 'Discourse', options)
+                        linuxConnection.restartUserService(USER, 'Discourse-Sidekiq', options)
+
+                        Nginx.withConnection(linuxConnection) do |nginxConnection|
+                            nginxConnection.provision(__dir__, 'Discourse', target, activeState, context, options)
+                        end
+
+                        linuxConnection.withUserShell(USER) do |shellConnection|
+                            Podman.withConnection(shellConnection, Podman.container(CONTAINER_NAME, shellConnection, { **options, 'dry' => false })) do |podmanConnection|
+                                if !target['Plugins'].to_a.empty?
+                                    target['Plugins'].each do |plugin|
+                                        podmanConnection.exec("RAILS_ENV=production bundle exec rake plugin:install repo=#{plugin}", true, { **options, workdir: '/opt/bitnami/discourse' })
+                                    end
+                                end
+
+                                waitForMigrations(podmanConnection, options)
+                                podmanConnection.exec('RAILS_ENV=production CHEAP_SOURCE_MAPS=1 JOBS=1 bundle exec rake assets:precompile', false, { **options, workdir: '/opt/bitnami/discourse' })
+                            end
+                        end
+                    end
+                end
+            end
+
+            def waitForMigrations(podmanConnection, options)
+                if !options['dry']
+                    timeout = 3600 # 1h
+                    loop do
+                        result = podmanConnection.exec('RAILS_ENV=production bundle exec rake db:migrate:status', false, { **options, workdir: '/opt/bitnami/discourse' })
+                        break unless result.include?('  down  ')
+                        timeout -= 30
+                        raise "Timeout while waiting for Discourse migrations to complete!" if timeout <= 0
+                        sleep(30)
+                    end
+                end
+            end
+
+            def configurePostgreSQL(dbSettings, linuxConnection, options)
+                password = SecureRandom.alphanumeric(20)
+                PostgreSQL.withConnection(dbSettings, linuxConnection) do |postgresConnection|
+                    postgresConnection.createUserAndDB(USER, password, options)
+                    postgresConnection.createExtensions(USER, ['hstore', 'pg_trgm'], options)
+                end
+                password
+            end
+
+        end
+    end
+end
+