mahout 1.1.0

data/lib/mahout/setup/postgres.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Mahout
+  module Setup
+    class Postgres
+      def call(runner, config, tuned)
+        stop_existing(runner, config)
+        init_cluster(runner, config)
+        render_config(runner, config, tuned)
+        start_postgres(runner, config)
+        create_tablespace(runner, config)
+        create_database(runner, config)
+        create_user(runner, config)
+        set_connection_limit(runner, config, tuned)
+      end
+
+      private
+
+      def stop_existing(runner, config)
+        runner.run("systemctl stop postgresql", allow_failure: true)
+      end
+
+      def init_cluster(runner, config)
+        result = runner.run("test -f #{config.pg_data_dir}/PG_VERSION", allow_failure: true)
+        return if result.success?
+
+        runner.run("pg_dropcluster --stop #{config.pg_version} main", allow_failure: true)
+        runner.run("find #{config.data_mount}/tablespace -mindepth 1 -delete", allow_failure: true)
+
+        waldir_flag = config.pg_wal_dir ? "--waldir=#{config.pg_wal_dir}" : ""
+        runner.run(
+          "pg_createcluster #{config.pg_version} main " \
+          "--datadir=#{config.pg_data_dir} " \
+          "-- --auth-local=peer --auth-host=scram-sha-256 #{waldir_flag}"
+        )
+      end
+
+      def render_config(runner, config, tuned)
+        tuned ||= {}
+        extensions = ExtensionRegistry.resolve(config.extensions, pg_version: config.pg_version)
+        preload = ExtensionRegistry.preload_libraries(extensions)
+        preload << "pg_prewarm" unless preload.include?("pg_prewarm")
+
+        runner.render_and_upload(
+          "postgresql.conf.erb",
+          "/etc/postgresql/#{config.pg_version}/main/postgresql.conf",
+          locals: {
+            config: config,
+            tuned: tuned,
+            preload_libraries: preload
+          },
+          owner: "postgres:postgres"
+        )
+
+        runner.run("mkdir -p /etc/postgresql/#{config.pg_version}/main/conf.d")
+        runner.run("chown postgres:postgres /etc/postgresql/#{config.pg_version}/main/conf.d")
+      end
+
+      def start_postgres(runner, config)
+        runner.run("systemctl enable postgresql")
+        runner.run("systemctl start postgresql", allow_failure: true)
+        sleep(2) unless runner.dry_run?
+
+        ready = runner.run("pg_isready --timeout=10", allow_failure: true)
+        return if ready.success?
+
+        pg_log = runner.run("tail -10 /var/log/postgresql/postgresql-#{config.pg_version}-main.log", allow_failure: true)
+
+        if pg_log.stdout.include?("huge pages") || pg_log.stdout.include?("Cannot allocate memory")
+          $stdout.puts("postgres failed to start with huge_pages = on, falling back to huge_pages = try")
+          conf_path = "/etc/postgresql/#{config.pg_version}/main/postgresql.conf"
+          runner.run("sed -i 's/huge_pages = on/huge_pages = try/' #{conf_path}")
+          runner.run("systemctl restart postgresql", allow_failure: true)
+          sleep(2) unless runner.dry_run?
+          ready = runner.run("pg_isready --timeout=30", allow_failure: true)
+          raise RemoteError, "postgresql failed to start even with huge_pages = try\n#{pg_log.stdout}" unless ready.success?
+        else
+          raise RemoteError, "postgresql failed to start\n#{pg_log.stdout}"
+        end
+      end
+
+      def create_tablespace(runner, config)
+        result = runner.run(
+          "sudo -u postgres psql -tAc \"SELECT 1 FROM pg_tablespace WHERE spcname = 'mahout_data'\"",
+          sudo: false, allow_failure: true
+        )
+        return if result.stdout.strip == "1"
+
+        runner.run("find #{config.data_mount}/tablespace -mindepth 1 -delete", allow_failure: true)
+        runner.run("mkdir -p #{config.data_mount}/tablespace")
+        runner.run("chown postgres:postgres #{config.data_mount}/tablespace")
+        runner.run(
+          "sudo -u postgres psql -c \"CREATE TABLESPACE mahout_data LOCATION '#{config.data_mount}/tablespace'\"",
+          sudo: false
+        )
+      end
+
+      def create_database(runner, config)
+        runner.run(
+          "sudo -u postgres psql -tc " \
+          "\"SELECT 1 FROM pg_database WHERE datname = '#{config.pg_database}'\" | grep -q 1 || " \
+          "sudo -u postgres createdb #{config.pg_database} --tablespace=mahout_data",
+          sudo: false
+        )
+      end
+
+      def create_user(runner, config)
+        password = config.secret("pg_password")
+
+        result = runner.run(
+          "sudo -u postgres psql -tAc \"SELECT 1 FROM pg_roles WHERE rolname = '#{config.pg_username}'\"",
+          sudo: false, allow_failure: true
+        )
+
+        unless result.stdout.strip == "1"
+          escaped_password = password.to_s.gsub("'", "''")
+          sql = "CREATE USER #{config.pg_username} WITH PASSWORD '#{escaped_password}';"
+          runner.upload(sql, "/tmp/mahout-create-user.sql", mode: "0600", owner: "postgres:postgres")
+          runner.run("sudo -u postgres psql -f /tmp/mahout-create-user.sql", sudo: false)
+          runner.run("rm -f /tmp/mahout-create-user.sql")
+        end
+
+        runner.run(
+          "sudo -u postgres psql -c " \
+          "\"GRANT ALL PRIVILEGES ON DATABASE #{config.pg_database} TO #{config.pg_username}\"",
+          sudo: false
+        )
+      end
+
+      def set_connection_limit(runner, config, tuned)
+        max_conn = tuned&.dig(:max_connections) || 200
+        app_limit = (max_conn * 0.8).to_i
+        runner.run(
+          "sudo -u postgres psql -c \"ALTER USER #{config.pg_username} CONNECTION LIMIT #{app_limit}\"",
+          sudo: false
+        )
+      end
+    end
+  end
+end

data/lib/mahout/setup/ssl.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Mahout
+  module Setup
+    class Ssl
+      def call(runner, config, _tuned)
+        cert_dir = "/etc/postgresql/#{config.pg_version}/main"
+        pg_conf = "#{cert_dir}/postgresql.conf"
+
+        runner.run(
+          "openssl req -new -x509 -days 3650 -nodes " \
+          "-out #{cert_dir}/server.crt " \
+          "-keyout #{cert_dir}/server.key " \
+          "-subj /CN=#{config.instance_name}"
+        )
+        runner.run("chmod 0600 #{cert_dir}/server.key")
+        runner.run("chown postgres:postgres #{cert_dir}/server.crt #{cert_dir}/server.key")
+
+        ssl_conf = <<~CONF
+          ssl = on
+          ssl_cert_file = '#{cert_dir}/server.crt'
+          ssl_key_file = '#{cert_dir}/server.key'
+        CONF
+        runner.upload(ssl_conf, "/etc/postgresql/#{config.pg_version}/main/conf.d/ssl.conf", owner: "postgres:postgres")
+        runner.run("systemctl reload postgresql")
+      end
+    end
+  end
+end

data/lib/mahout/setup/systemd.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+module Mahout
+  module Setup
+    class Systemd
+      def call(runner, config, _tuned)
+        if config.pgbackrest_enabled?
+          install_backup_units(runner, config)
+          install_verify_timer(runner, config)
+          install_wal_check(runner, config)
+        end
+        install_health_check(runner, config)
+        install_oom_override(runner, config)
+        install_stats_reset_timer(runner)
+        runner.run("systemctl daemon-reload")
+        enable_timers(runner, config)
+      end
+
+      private
+
+      def install_backup_units(runner, config)
+        stanza = config.pgbackrest_stanza
+
+        %w[full diff incr].each do |type|
+          service = <<~UNIT
+            [Unit]
+            Description=pgBackRest #{type} backup
+            After=postgresql.service
+
+            [Service]
+            Type=oneshot
+            User=postgres
+            ExecStart=/usr/bin/pgbackrest --stanza=#{stanza} --type=#{type} backup
+          UNIT
+
+          calendar = case type
+                      when "full" then config.pgbackrest_schedule_full
+                      when "diff" then config.pgbackrest_schedule_diff
+                      when "incr" then config.pgbackrest_schedule_incr
+                      end
+
+          timer = <<~UNIT
+            [Unit]
+            Description=pgBackRest #{type} backup timer
+
+            [Timer]
+            OnCalendar=#{calendar}
+            Persistent=true
+
+            [Install]
+            WantedBy=timers.target
+          UNIT
+
+          runner.upload(service, "/etc/systemd/system/pgbackrest-#{type}-backup.service")
+          runner.upload(timer, "/etc/systemd/system/pgbackrest-#{type}-backup.timer")
+        end
+      end
+
+      def install_verify_timer(runner, config)
+        stanza = config.pgbackrest_stanza
+
+        service = <<~UNIT
+          [Unit]
+          Description=pgBackRest verify
+          After=postgresql.service
+
+          [Service]
+          Type=oneshot
+          User=postgres
+          ExecStart=/usr/bin/pgbackrest --stanza=#{stanza} verify
+        UNIT
+
+        timer = <<~UNIT
+          [Unit]
+          Description=pgBackRest weekly verify timer
+
+          [Timer]
+          OnCalendar=Sun *-*-* 04:00:00
+          Persistent=true
+
+          [Install]
+          WantedBy=timers.target
+        UNIT
+
+        runner.upload(service, "/etc/systemd/system/pgbackrest-verify.service")
+        runner.upload(timer, "/etc/systemd/system/pgbackrest-verify.timer")
+      end
+
+      def install_wal_check(runner, config)
+        stanza = config.pgbackrest_stanza
+
+        service = <<~UNIT
+          [Unit]
+          Description=pgBackRest WAL archive check
+          After=postgresql.service
+
+          [Service]
+          Type=oneshot
+          User=postgres
+          ExecStart=/usr/bin/pgbackrest --stanza=#{stanza} check
+        UNIT
+
+        timer = <<~UNIT
+          [Unit]
+          Description=pgBackRest WAL archive check timer
+
+          [Timer]
+          OnCalendar=*-*-* *:00/5:00
+          Persistent=true
+
+          [Install]
+          WantedBy=timers.target
+        UNIT
+
+        runner.upload(service, "/etc/systemd/system/pgbackrest-wal-check.service")
+        runner.upload(timer, "/etc/systemd/system/pgbackrest-wal-check.timer")
+      end
+
+      def install_health_check(runner, config)
+        service = <<~UNIT
+          [Unit]
+          Description=mahout health check
+          After=postgresql.service
+
+          [Service]
+          Type=oneshot
+          ExecStart=/bin/bash -c 'pg_isready -q && echo ok > /var/log/mahout/health.status || echo fail > /var/log/mahout/health.status'
+        UNIT
+
+        timer = <<~UNIT
+          [Unit]
+          Description=mahout health check timer
+
+          [Timer]
+          OnCalendar=*-*-* *:*:00
+          Persistent=true
+
+          [Install]
+          WantedBy=timers.target
+        UNIT
+
+        runner.run("mkdir -p /var/log/mahout")
+        runner.upload(service, "/etc/systemd/system/pg-health-check.service")
+        runner.upload(timer, "/etc/systemd/system/pg-health-check.timer")
+      end
+
+      def install_oom_override(runner, config)
+        runner.run("mkdir -p /etc/systemd/system/postgresql.service.d")
+        dropin = <<~UNIT
+          [Service]
+          OOMScoreAdjust=-1000
+        UNIT
+        runner.upload(dropin, "/etc/systemd/system/postgresql.service.d/oom.conf")
+      end
+
+      def install_stats_reset_timer(runner)
+        service = <<~UNIT
+          [Unit]
+          Description=Reset pg_stat_statements
+          After=postgresql.service
+
+          [Service]
+          Type=oneshot
+          ExecStart=/usr/bin/sudo -u postgres psql -c "SELECT pg_stat_statements_reset()"
+        UNIT
+
+        timer = <<~UNIT
+          [Unit]
+          Description=Weekly pg_stat_statements reset
+
+          [Timer]
+          OnCalendar=Mon *-*-* 03:00:00
+          Persistent=true
+
+          [Install]
+          WantedBy=timers.target
+        UNIT
+
+        runner.upload(service, "/etc/systemd/system/pg-stats-reset.service")
+        runner.upload(timer, "/etc/systemd/system/pg-stats-reset.timer")
+      end
+
+      def enable_timers(runner, config)
+        timers = %w[
+          pg-health-check.timer
+          pg-stats-reset.timer
+        ]
+
+        if config.pgbackrest_enabled?
+          timers.unshift(
+            "pgbackrest-full-backup.timer",
+            "pgbackrest-diff-backup.timer",
+            "pgbackrest-incr-backup.timer",
+            "pgbackrest-verify.timer",
+            "pgbackrest-wal-check.timer"
+          )
+        end
+
+        timers.each do |timer|
+          runner.run("systemctl enable --now #{timer}")
+        end
+      end
+    end
+  end
+end

data/lib/mahout/status.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Mahout
+  class Status
+    def initialize(runner:, config:)
+      @runner = runner
+      @config = config
+    end
+
+    def call
+      show_pg_info
+      show_disk_usage
+      show_connections
+      show_backup_info if @config.pgbackrest_enabled?
+      show_pgbouncer_stats
+      show_extensions
+      show_timer_status
+    end
+
+    private
+
+    def show_pg_info
+      result = @runner.run(
+        "sudo -u postgres psql -tAc \"SELECT version()\"",
+        sudo: false, allow_failure: true
+      )
+      $stdout.puts("postgresql: #{result.stdout.strip.split(" ").first(2).join(" ")}") if result.success?
+
+      result = @runner.run(
+        "sudo -u postgres psql -tAc \"SELECT now() - pg_postmaster_start_time()\"",
+        sudo: false, allow_failure: true
+      )
+      $stdout.puts("uptime: #{result.stdout.strip}") if result.success?
+    end
+
+    def show_disk_usage
+      [@config.data_mount, @config.wal_mount].compact.each do |mount|
+        result = @runner.run("df -h --output=size,used,avail,pcent #{mount} | tail -1", allow_failure: true)
+        $stdout.puts("disk #{mount}: #{result.stdout.strip}") if result.success?
+      end
+    end
+
+    def show_connections
+      result = @runner.run(
+        "sudo -u postgres psql -tAc \"SELECT count(*) || '/' || current_setting('max_connections') FROM pg_stat_activity\"",
+        sudo: false, allow_failure: true
+      )
+      $stdout.puts("connections: #{result.stdout.strip}") if result.success?
+    end
+
+    def show_backup_info
+      result = @runner.run(
+        "sudo -u postgres pgbackrest --stanza=#{@config.pgbackrest_stanza} info --output json",
+        sudo: false, allow_failure: true
+      )
+      return unless result.success? && !result.stdout.strip.empty?
+
+      info = JSON.parse(result.stdout) rescue return
+      backups = info.dig(0, "backup")
+      return $stdout.puts("backups: none") if backups.nil? || backups.empty?
+
+      latest = backups.last
+      $stdout.puts("last backup: #{latest["type"]} at #{Time.at(latest.dig("timestamp", "stop")).utc}")
+      $stdout.puts("total backups: #{backups.length}")
+    end
+
+    def show_pgbouncer_stats
+      result = @runner.run(
+        "psql -h 127.0.0.1 -p #{@config.pgbouncer_port} -U #{@config.pg_username} pgbouncer -tAc 'SHOW POOLS' 2>&1",
+        sudo: false, allow_failure: true
+      )
+      if result.success?
+        $stdout.puts("pgbouncer: active")
+      else
+        $stdout.puts("pgbouncer: not responding")
+      end
+    end
+
+    def show_extensions
+      result = @runner.run(
+        "sudo -u postgres psql -d #{@config.pg_database} -tAc \"SELECT extname || ' ' || extversion FROM pg_extension ORDER BY extname\"",
+        sudo: false, allow_failure: true
+      )
+      return unless result.success?
+
+      exts = result.stdout.strip.split("\n").reject(&:empty?)
+      $stdout.puts("extensions: #{exts.join(", ")}")
+    end
+
+    def show_timer_status
+      result = @runner.run("systemctl list-timers --no-pager pgbackrest-* pg-* 2>/dev/null | head -10", allow_failure: true)
+      return unless result.success? && !result.stdout.strip.empty?
+
+      $stdout.puts("")
+      $stdout.puts("timers:")
+      $stdout.puts(result.stdout.strip)
+    end
+  end
+end