caco 0.1.0

data/lib/caco/prometheus/view/alertmanager_conf.erb

@@ -0,0 +1,13 @@
+global:
+  slack_api_url: 'https://hooks.slack.com/services/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'
+
+route:
+  receiver: 'slack-notifications'
+  group_by: [alertname, datacenter, app]
+
+receivers:
+- name: 'slack-notifications'
+  slack_configs:
+  - send_resolved: true
+  - channel: '#monitoring'
+    text: 'https://internal.myorg.net/wiki/alerts/{{ .GroupLabels.app }}/{{ .GroupLabels.alertname }}'

data/lib/caco/prometheus/view/alerts.erb

@@ -0,0 +1,18 @@
+groups:
+- name: alert.rules
+  rules:
+  - alert: EndpointDown
+    expr: probe_success == 0
+    for: 10s
+    labels:
+      severity: 'critical'
+    annotations:
+      summary: 'Endpoint down'
+  - alert: ExporterDown
+    expr: up == 0
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: 'Exporter down (instance {{ $labels.instance }})'
+      description: 'Prometheus exporter down\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}'

data/lib/caco/prometheus/view/conf.erb

@@ -0,0 +1,34 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+remote_write:
+  - url: "http://127.0.0.1:9201/write"
+remote_read:
+  - url: "http://127.0.0.1:9201/read"
+
+rule_files:
+  - <%= root %>/alerts.d/*.rules
+
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+      - localhost:9093
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']
+  - job_name: postgresql
+    static_configs:
+      - targets: ['localhost:9187']
+  - job_name: postgresql_adapter
+    static_configs:
+      - targets: ['localhost:9201']
+  - job_name: node_exporter
+    scrape_interval: 1m
+    scrape_timeout:  1m
+    metrics_path: '/metrics'
+    static_configs: 
+      - targets: ['localhost:9100']

data/lib/caco/rbenv.rb

@@ -0,0 +1,8 @@
+module Caco
+  module Rbenv
+  end
+end
+
+require 'caco/rbenv/install'
+require 'caco/rbenv/install_version'
+require 'caco/rbenv/cell/profile'

data/lib/caco/rbenv/cell/profile.rb

@@ -0,0 +1,4 @@
+module Caco::Rbenv::Cell
+  class Profile < Trailblazer::Cell
+  end
+end

data/lib/caco/rbenv/install.rb

@@ -0,0 +1,56 @@
+module Caco::Rbenv
+  class Install < Trailblazer::Operation
+    CloneRepo = Class.new(Trailblazer::Activity::Signal)
+
+    step :install_packages!
+    step :repo_exist?, Output(CloneRepo, :clone_repo) => Path(connect_to: Id(:make_src)) do
+      step :clone_repo
+    end
+
+    pass Subprocess(Class.new(Caco::Executer)),
+      input: ->(_ctx, **) {{
+        command: "[ ! -f /opt/rbenv/libexec/rbenv-realpath.dylib ] && cd /opt/rbenv && src/configure && make -C src; true",
+      }},
+      Output(:failure) => Track(:success),
+      id: :make_src
+
+    step ->(ctx, **) {
+        ctx[:profile_content] = Caco::Rbenv::Cell::Profile.().to_s
+      },
+      id: :build_profile_content
+
+    step Subprocess(Caco::FileWriter),
+      input: ->(_ctx, profile_content:, **) {{
+        path: "/etc/profile.d/rbenv.sh",
+        content: profile_content
+      }}
+
+    def install_packages!(ctx, **)
+      packages = []
+      if Caco::Facter.("os", "distro", "codename") == "stretch"
+        packages = %w(git autoconf bison libssl-dev libyaml-dev libreadline-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm3 libgdbm-dev)
+      elsif Caco::Facter.("os", "distro", "codename") == "buster"
+        packages = %w(git autoconf bison libssl-dev libyaml-dev libreadline-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm6 libgdbm-dev)
+      end
+
+      packages.each do |package|
+        result = Caco::Debian::PackageInstall.(package: package)
+        return false if result.failure?
+      end
+      true
+    end
+
+    def repo_exist?(ctx, **)
+      self.class.repo_exist?
+    end
+
+    def self.repo_exist?
+      File.exist?("/opt/rbenv") ? true : CloneRepo
+    end
+
+    def clone_repo(ctx, **)
+      result = Caco::Executer.(command: "git clone https://github.com/rbenv/rbenv.git /opt/rbenv")
+      result.success?
+    end
+  end
+end

data/lib/caco/rbenv/install_version.rb

@@ -0,0 +1,17 @@
+module Caco::Rbenv
+  class InstallVersion < Trailblazer::Operation
+    step Subprocess(Class.new(Caco::Executer)),
+      input: ->(_ctx, version:, **) {{
+        command: ". /etc/profile && /opt/rbenv/bin/rbenv versions|egrep --color \"^..#{version}(\s|$)\"",
+      }},
+      Output(:failure) => Id(:install_version),
+      id: :install_version_checker
+
+    step Subprocess(Class.new(Caco::Executer)),
+      input: ->(_ctx, version:, **) {{
+        command: ". /etc/profile && /opt/rbenv/bin/rbenv install #{version}",
+      }},
+      id: :install_version,
+      magnetic_to: nil
+  end
+end

data/lib/caco/rbenv/view/profile.erb

@@ -0,0 +1,3 @@
+export PATH="/opt/rbenv/bin:$PATH"
+export RBENV_ROOT="/opt/rbenv"
+eval "$(rbenv init -)"

data/lib/caco/repmgr.rb

@@ -0,0 +1,15 @@
+module Caco
+  module Repmgr
+  end
+end
+
+require 'caco/repmgr/conf'
+require 'caco/repmgr/install'
+require 'caco/repmgr/node_registered'
+require 'caco/repmgr/node_role'
+
+require 'caco/repmgr/node_register_primary'
+require 'caco/repmgr/node_register_standby'
+
+# Templates
+require 'caco/repmgr/cell/conf'

data/lib/caco/repmgr/cell/conf.rb

@@ -0,0 +1,43 @@
+module Caco::Repmgr::Cell
+  class Conf < Trailblazer::Cell
+    def node_id
+      property(:node_id)
+    end
+
+    def node_name
+      property(:node_name)
+    end
+
+    def primary_host
+      property(:primary_host)
+    end
+
+    def primary_user
+      property(:primary_user)
+    end
+
+    def primary_database
+      property(:primary_database)
+    end
+
+    def node_initial_role
+      property(:node_initial_role)
+    end
+
+    def postgres_version
+      property(:postgres_version)
+    end
+
+    def use_barman?
+      property(:use_barman) || false
+    end
+
+    def barman_host
+      property(:barman_host) || "barman"
+    end
+
+    def barman_server
+      property(:barman_server) || "pg"
+    end
+  end
+end

data/lib/caco/repmgr/conf.rb

@@ -0,0 +1,25 @@
+module Caco::Repmgr
+  class Conf < Trailblazer::Operation
+    step ->(ctx, **) {
+        ctx[:path] = "/etc/repmgr.conf"
+      },
+      id: :build_path
+
+    step :build_content
+
+    step Subprocess(Caco::FileWriter),
+      input: ->(_ctx, path:, content:, **) {{
+        path: path,
+        content: content
+      }},
+      output: {file_created: :created, file_changed: :changed}
+
+    def build_content(ctx, node_id:, node_name:, postgres_version:, **)
+      ctx[:content] = Caco::Repmgr::Cell::Conf.(
+        node_id: node_id,
+        node_name: node_name,
+        postgres_version: postgres_version
+      ).to_s
+    end
+  end
+end

data/lib/caco/repmgr/install.rb

@@ -0,0 +1,25 @@
+module Caco::Repmgr
+  class Install < Trailblazer::Operation
+    class Repo < Trailblazer::Operation
+      step Subprocess(Caco::Debian::AptKeyInstall),
+        input: ->(_ctx, **) {{
+          url: 'https://dl.2ndquadrant.com/gpg-key.asc',
+          fingerprint: '8565 305C EA7D 0B66 4933  D250 9904 CD4B D6BA F0C3'
+        }}
+      step Subprocess(Caco::Debian::AptRepoAdd),
+        input: ->(_ctx, **) {{
+          name: '2ndquadrant-dl-default-release',
+          url: 'https://dl.2ndquadrant.com/default/release/apt',
+          release: "#{Caco::Facter.("os", "distro", "codename")}-2ndquadrant",
+          component: 'main'
+        }}
+    end
+
+    step Subprocess(Repo)
+    step Subprocess(Caco::Debian::AptUpdate)
+    step Subprocess(Caco::Debian::PackageInstall),
+      input: ->(_ctx, **) {{
+        package: 'repmgr'
+      }}
+  end
+end

data/lib/caco/repmgr/node_register_primary.rb

@@ -0,0 +1,34 @@
+module Caco::Repmgr
+  class NodeRegisterPrimary < Trailblazer::Operation
+    step Subprocess(Caco::Repmgr::NodeRole),
+      input: ->(_ctx, node_name:, **) {{
+        node_name: node_name
+      }},
+      id: :node_role,
+      Output(:success) => Id(:check_existing_id),
+      Output(:failure) => Track(:success)
+
+    step :check_existing_id, magnetic_to: nil,
+      Output(:success) => End(:success),
+      Output(:failure) => End(:failure)
+
+    step :verify_any_primary,
+      Output(:success) => End(:failure),
+      Output(:failure) => Track(:success)
+
+    step Subprocess(Class.new(Caco::Executer)),
+      input: ->(_ctx, node_name:, **) {{
+        command: "su - postgres -c 'repmgr primary register'"
+      }},
+      id: :repmgr_register_primary
+
+    def check_existing_id(ctx, node_role:, **)
+      return true if node_role == "primary"
+      false
+    end
+
+    def verify_any_primary(ctx, node_name:, output:, **)
+      output.match?(/^\s+[0-9]{1,}\s+\|\s(?!#{node_name})[^\s]+\s+\|\s+primary\s+\|/)
+    end
+  end
+end

data/lib/caco/repmgr/node_register_standby.rb

@@ -0,0 +1,25 @@
+module Caco::Repmgr
+  class NodeRegisterStandby < Trailblazer::Operation
+    step Subprocess(Caco::Repmgr::NodeRole),
+      input: ->(_ctx, node_name:, **) {{
+        node_name: node_name
+      }},
+      id: :node_role,
+      Output(:success) => Id(:check_existing_id),
+      Output(:failure) => Track(:success)
+
+    step ->(ctx, node_role:, **) {
+        node_role == "standby"
+      },
+      magnetic_to: nil,
+      Output(:success) => End(:success),
+      Output(:failure) => End(:failure),
+      id: :check_existing_id
+
+    step Subprocess(Class.new(Caco::Executer)),
+      input: ->(_ctx, node_name:, **) {{
+        command: "su - postgres -c 'repmgr standby register'"
+      }},
+      id: :repmgr_register_primary
+  end
+end

data/lib/caco/repmgr/node_registered.rb

@@ -0,0 +1,15 @@
+module Caco::Repmgr
+  class NodeRegistered < Trailblazer::Operation
+    step Subprocess(Caco::Executer),
+      input: ->(_ctx, node_name:, **) {{
+        command: "su - postgres -c 'repmgr cluster show --compact'"
+      }},
+      id: :repmgr_cluster_show
+
+    step ->(ctx, node_name:, output:, **) {
+        # set to ctx so can be used in other operations
+        ctx[:node_registered] = output.match?(/^\s+[0-9]{1,}\s+\|\s+#{node_name}\s+\|/)
+      },
+      id: :verify_node
+  end
+end

data/lib/caco/repmgr/node_role.rb

@@ -0,0 +1,18 @@
+module Caco::Repmgr
+  class NodeRole < Trailblazer::Operation
+    step Subprocess(NodeRegistered),
+      input: ->(_ctx, node_name:, **) {{
+        node_name: node_name
+      }},
+      id: :node_registered
+
+    step :verify_role
+
+    def verify_role(ctx, node_name:, output:, **)
+      match = output.match(/^\s+[0-9]{1,}\s+\|\s+db1\s+\|\s+([^\s]+)\s+\|/)
+      return false unless match
+
+      ctx[:node_role] = match[1]
+    end
+  end
+end

data/lib/caco/repmgr/view/conf.erb

@@ -0,0 +1,27 @@
+node_id=<%= node_id %>
+node_name='<%= node_name %>'
+conninfo='host=<%= primary_host %> user=<%= primary_user %> dbname=<%= primary_database %> connect_timeout=2'
+data_directory='/var/lib/postgresql/<%= postgres_version %>/main'
+
+use_replication_slots=yes
+monitoring_history=yes
+
+service_start_command   = 'sudo /usr/bin/pg_ctlcluster <%= postgres_version %> main start'
+service_stop_command    = 'sudo /usr/bin/pg_ctlcluster <%= postgres_version %> main stop'
+service_restart_command = 'sudo /usr/bin/pg_ctlcluster <%= postgres_version %> main restart'
+service_reload_command  = 'sudo /usr/bin/pg_ctlcluster <%= postgres_version %> main reload'
+service_promote_command = 'sudo /usr/bin/pg_ctlcluster <%= postgres_version %> main promote'
+
+promote_check_timeout = 15
+
+failover=automatic
+promote_command='/usr/bin/repmgr standby promote -f /etc/repmgr.conf --log-to-file'
+follow_command='/usr/bin/repmgr standby follow -f /etc/repmgr.conf --log-to-file --upstream-node-id=%n'
+
+log_file='/var/log/postgresql/repmgrd.log'
+
+<% if use_barman? %>
+barman_host='<%= barman_user %>@<%= barman_host %>'
+barman_server='<%= barman_host %>'
+restore_command='/usr/bin/barman-wal-restore <%= barman_host %> <%= barman_server %> %f %p'
+<% end %>

data/lib/caco/settings_loader.rb

@@ -0,0 +1,67 @@
+require_relative "./settings_loader_monkeypatch"
+
+module Caco
+  class SettingsLoader < Trailblazer::Operation
+    step :setup_validate_params
+    step :config_setup
+    step :hiera_setup_keys
+    step :set_caco_eyaml_parser
+    step :facter_needed_values
+    step :config_load
+    step :custom_config
+
+    def setup_validate_params(ctx, keys_path: nil, data_path: nil, **)
+      ctx[:keys_path] = !keys_path.nil? ? Pathname.new(keys_path) : Pathname.new(Caco.root.join("keys"))
+      ctx[:data_path] = !data_path.nil? ? Pathname.new(data_path) : Pathname.new(Caco.root.join("data"))
+    end
+
+    def config_setup(ctx, **)
+      Config.setup do |config|
+        config.const_name = 'Settings'
+        config.use_env = true
+      end
+    end
+
+    def hiera_setup_keys(ctx, keys_path:, **)
+      Hiera::Backend::Eyaml::Options[:pkcs7_public_key] = keys_path.join("public_key.pkcs7.pem")
+      Hiera::Backend::Eyaml::Options[:pkcs7_private_key] = keys_path.join("private_key.pkcs7.pem")
+      ctx[:parser] = Hiera::Backend::Eyaml::Parser::ParserFactory.encrypted_parser
+    end
+
+    def set_caco_eyaml_parser(ctx, parser:, **)
+      Caco.configure do |config|
+        config.eyaml_parser = parser
+      end
+    end
+
+    def facter_needed_values(ctx, **)
+      ctx[:facts] = {}
+      ctx[:facts][:facter_kernel] =  Caco::Facter.("kernel")
+      ctx[:facts][:facter_os_name] = Caco::Facter.("os", "name")
+      ctx[:facts][:facter_distro_codename] = (ctx[:facts][:facter_kernel] == "Linux" ? Caco::Facter.("os", "distro", "codename") : nil)
+      ctx[:facts][:facter_release_full] = Caco::Facter.("os", "release", "full")
+      ctx[:facts][:facter_release_major] = Caco::Facter.("os", "release", "major")
+      ctx[:facts][:facter_release_minor] = Caco::Facter.("os", "release", "minor")
+      ctx[:facts][:facter_fqdn] = Caco::Facter.("networking", "fqdn")
+    end
+
+    def config_load(ctx, facts:, data_path:, **)
+      # From more generic to specific
+      Config.load_and_set_settings(
+        data_path.join("common.yaml"),
+        data_path.join("os", "#{facts[:facter_os_name]}.yaml"),
+        data_path.join("os", "#{facts[:facter_os_name]}", "#{facts[:facter_distro_codename]}.yaml"),
+        # maybe add some organizations here?
+        # maybe add some roles here?
+        data_path.join("nodes", "#{facts[:facter_fqdn]}"),
+      )
+      Settings.reload!
+    end
+
+    def custom_config(ctx, **)
+      Settings.prometheus = Config::Options.new
+      Settings.prometheus.root = "/opt/prometheus"
+      Settings.prometheus.config_root = "/etc/prometheus"
+    end
+  end
+end