renuo-cli 4.26.0 → 4.28.0

data/lib/renuo/cli/services/deploio/copy_s3_data.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative "create_s3"
+
+class Renuo::Cli::Services::Deploio
+  class CopyS3Data < Actor
+    include ReviewAppHelper
+
+    DATA_DIR = File.join(Dir.tmpdir, "review-app-bucket-data")
+    LISTING_ATTEMPTS = 24
+    LISTING_INTERVAL_IN_SECONDS = 5
+
+    input :target_app_name, must: satisfy(SafeReviewAppNameContract)
+    input :s3_env
+    input :origin_bucket_name
+    input :origin_bucket_endpoint
+    input :origin_bucket_access_key
+    input :origin_bucket_secret_key
+
+    def call
+      return puts "bucket #{target_app_name} already holds objects, skipping data copy" unless bucket_empty?
+
+      puts "copying bucket data from #{origin_bucket_name} into #{target_app_name}..."
+      download_data
+      upload_data
+    end
+
+    private
+
+    def download_data
+      system! "aws s3 sync s3://#{origin_bucket_name} #{DATA_DIR} --endpoint-url https://#{origin_bucket_endpoint}",
+              aws_credentials(origin_bucket_access_key, origin_bucket_secret_key)
+    end
+
+    # aws exits non-zero on some empty listings, so only stderr tells a failure apart.
+    def bucket_empty?
+      stdout, stderr = list_target_bucket
+      abort "'aws s3' failed: #{stderr}" unless stderr.strip.empty?
+      stdout.strip.empty?
+    end
+
+    # Deploio reports a fresh bucket ready seconds before its permissions grant access.
+    def list_target_bucket
+      (1..LISTING_ATTEMPTS).each do |attempt|
+        stdout, stderr, = system!("aws s3 ls s3://#{target_app_name} --endpoint-url #{target_endpoint}",
+                                  target_credentials, quiet: true, abort_on_failure: false)
+        return [stdout, stderr] if attempt == LISTING_ATTEMPTS || stderr.exclude?("AccessDenied")
+
+        puts "waiting for access to bucket #{target_app_name}..."
+        Kernel.sleep(LISTING_INTERVAL_IN_SECONDS)
+      end
+    end
+
+    def upload_data
+      system! "aws s3 sync #{DATA_DIR} s3://#{target_app_name} --endpoint-url #{target_endpoint}", target_credentials
+    end
+
+    def target_endpoint
+      s3_env.fetch(target_schema[:endpoint])
+    end
+
+    def target_credentials
+      aws_credentials(s3_env.fetch(target_schema[:access_key]), s3_env.fetch(target_schema[:secret_key]))
+    end
+
+    def target_schema
+      CreateS3::ENV_SCHEMAS_IN_THE_WILD.find { |candidate| s3_env.key?(candidate[:secret_key]) }
+    end
+  end
+end

data/lib/renuo/cli/services/deploio/create_database.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "shellwords"
+
+class Renuo::Cli::Services::Deploio
+  class CreateDatabase < Actor
+    include ReviewAppHelper
+    include Renuo::Cli::Services
+
+    input :project
+    input :target_app_name, must: satisfy(SafeReviewAppNameContract)
+    input :origin_database_location, allow_nil: true, default: nil
+    input :origin_database_version, allow_nil: true, default: nil
+    input :origin_database_collation, allow_nil: true, default: nil
+
+    output :database_env
+
+    def call
+      if Deploio.resource_exists?("postgresdatabase", target_app_name, project)
+        puts "database #{target_app_name} already exists, skipping creation"
+      else
+        create_database
+      end
+      self.database_env = { "DATABASE_URL" => database_url }
+    end
+
+    private
+
+    def create_database
+      puts "creating database #{target_app_name}..."
+      system! ["nctl create postgresdatabase #{target_app_name} -p #{project}",
+               *origin_flags,
+               "--backup-schedule=disabled --wait"].join(" ")
+    end
+
+    def origin_flags
+      { "--location" => origin_database_location,
+        "--postgres-database-version" => origin_database_version,
+        "--collation" => origin_database_collation }
+        .filter_map { |flag, value| "#{flag} #{Shellwords.escape(value)}" if value }
+    end
+
+    def database_url
+      stdout, = system!("nctl get postgresdatabase #{target_app_name} -p #{project} --print-connection-string",
+                        quiet: true)
+      stdout.strip
+    end
+  end
+end

data/lib/renuo/cli/services/deploio/create_kvs.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "shellwords"
+
+class Renuo::Cli::Services::Deploio
+  class CreateKvs < Actor
+    include ReviewAppHelper
+    include Renuo::Cli::Services
+
+    input :project
+    input :target_app_name, must: satisfy(SafeReviewAppNameContract)
+    input :origin_key_value_store_location, allow_nil: true, default: nil
+
+    output :kvs_env
+
+    def call
+      if Deploio.resource_exists?("kvs", target_app_name, project)
+        puts "key-value store #{target_app_name} already exists, skipping creation"
+      else
+        create_kvs
+      end
+      self.kvs_env = { "REDIS_URL" => redis_url }
+    end
+
+    private
+
+    def create_kvs
+      system! ["nctl create kvs #{target_app_name} -p #{project}", location_flag, "--wait"].compact.join(" ")
+    end
+
+    def location_flag
+      "--location #{Shellwords.escape(origin_key_value_store_location)}" if origin_key_value_store_location
+    end
+
+    def redis_url
+      token, = system!("nctl get kvs #{target_app_name} -p #{project} --print-token", quiet: true)
+      fqdn = Deploio.fetch_resource("kvs", target_app_name, project).dig("status", "atProvider", "fqdn")
+      "rediss://:#{token.strip}@#{fqdn}:6379"
+    end
+  end
+end

data/lib/renuo/cli/services/deploio/create_s3.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+class Renuo::Cli::Services::Deploio
+  class CreateS3 < Actor
+    include ReviewAppHelper
+    include Renuo::Cli::Services
+
+    DEFAULT_BUCKET_LOCATION = "nine-es34"
+    ENV_SCHEMAS_IN_THE_WILD = [
+      {
+        name: "DEPLOIO_BUCKET_NAME",
+        endpoint: "DEPLOIO_BUCKET_ENDPOINT",
+        access_key: "DEPLOIO_BUCKET_ACCESS_KEY",
+        secret_key: "DEPLOIO_BUCKET_SECRET_KEY"
+      },
+      {
+        name: "DEPLOIO_BUCKET",
+        endpoint: "DEPLOIO_ENDPOINT",
+        access_key: "DEPLOIO_ACCESS_KEY",
+        secret_key: "DEPLOIO_SECRET_KEY"
+      }
+    ].freeze
+    SENSITIVE_ENV_KEYS = ENV_SCHEMAS_IN_THE_WILD.flat_map { |schema| schema.values_at(:access_key, :secret_key) }.freeze
+
+    input :project
+    input :target_app_name, must: satisfy(SafeReviewAppNameContract)
+    input :template, default: {}
+
+    output :s3_env
+
+    def call
+      location = template[:bucket_location] || DEFAULT_BUCKET_LOCATION
+      create_bucket_user(location)
+      create_bucket(location)
+      self.s3_env = bucket_env
+    end
+
+    private
+
+    def app_env?(key)
+      (template[:app_env] || []).any? { |variable| variable["name"] == key }
+    end
+
+    def create_bucket_user(location)
+      return if Deploio.resource_exists?("bucketuser", target_app_name, project)
+
+      system! "nctl create bucketuser #{target_app_name} -p #{project} --location #{location} --wait"
+    end
+
+    def create_bucket(location)
+      if Deploio.resource_exists?("bucket", target_app_name, project)
+        return puts "bucket #{target_app_name} already exists, skipping creation"
+      end
+
+      system! "nctl create bucket #{target_app_name} -p #{project} --location #{location} " \
+              "--permissions \"reader=#{target_app_name};writer=#{target_app_name}\" --wait"
+    end
+
+    def bucket_env
+      schema = env_schema
+      {
+        schema[:name] => target_app_name,
+        schema[:endpoint] => "https://#{bucket_endpoint}",
+        schema[:access_key] => bucket_credential("--print-access-key"),
+        schema[:secret_key] => bucket_credential("--print-secret-key")
+      }
+    end
+
+    def bucket_endpoint
+      Deploio.fetch_resource("bucket", target_app_name, project).dig("status", "atProvider", "endpoint")
+    end
+
+    def bucket_credential(flag)
+      stdout, = system!("nctl get bucketuser #{target_app_name} -p #{project} #{flag}", quiet: true)
+      stdout.strip
+    end
+
+    def env_schema
+      ENV_SCHEMAS_IN_THE_WILD.find { |schema| app_env?(schema[:secret_key]) } || ENV_SCHEMAS_IN_THE_WILD.first
+    end
+  end
+end

data/lib/renuo/cli/services/deploio/inventorize_origin_resources.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+class Renuo::Cli::Services::Deploio
+  class InventorizeOriginResources < Actor # rubocop:disable Metrics/ClassLength
+    include ReviewAppHelper
+
+    input :project
+    input :origin_pattern,
+          must: {
+            be_specific_enough: {
+              is: ->(pattern) { pattern.to_s.length >= ReviewAppHelper::MIN_PATTERN_LENGTH },
+              message: (lambda do |value:, **|
+                "origin pattern '#{value}' must be at least #{ReviewAppHelper::MIN_PATTERN_LENGTH} " \
+                  "characters long, aborting"
+              end)
+            }
+          }
+
+    output :origin_app_name
+    output :origin_database_name
+    output :origin_database_location
+    output :origin_database_version
+    output :origin_database_collation
+    output :origin_backup_bucket
+    output :origin_backup_path
+    output :origin_backup_endpoint
+    output :origin_backup_bucket_user
+    output :origin_backup_access_key
+    output :origin_backup_secret_key
+    output :origin_key_value_store_name
+    output :origin_key_value_store_location
+    output :origin_bucket_name
+    output :origin_bucket_endpoint
+    output :origin_bucket_user
+    output :origin_bucket_access_key
+    output :origin_bucket_secret_key
+
+    def call
+      inventorize_app
+      inventorize_database
+      inventorize_backup
+      inventorize_key_value_store
+      inventorize_bucket
+      ensure_required_tools
+    end
+
+    private
+
+    def inventorize_app
+      self.origin_app_name = origin_app_name!
+    end
+
+    def inventorize_database
+      database = origin_resource("PostgresDatabase")
+      self.origin_database_name = database&.dig("metadata", "name")
+      self.origin_database_location, self.origin_database_version, self.origin_database_collation =
+        %w[location version collation].map { |key| database&.dig("spec", "forProvider", key)&.to_s }
+    end
+
+    def inventorize_backup
+      self.origin_backup_bucket = backup_bucket_name
+      self.origin_backup_path = latest_backup_path
+      backup_bucket = bucket_resource(origin_backup_bucket)
+      self.origin_backup_endpoint = backup_bucket&.dig("status", "atProvider", "endpoint")
+      self.origin_backup_bucket_user = bucket_user(backup_bucket)
+      self.origin_backup_access_key, self.origin_backup_secret_key =
+        bucketuser_credentials(origin_backup_bucket_user)
+    end
+
+    def bucket_user(bucket)
+      return nil unless bucket
+
+      bucket_reader_user(bucket) || abort("no bucket user for '#{bucket.dig("metadata", "name")}' in #{project}")
+    end
+
+    # Bucket users are not necessarily named like their bucket; the bucket's permissions reference them.
+    def bucket_reader_user(bucket)
+      permissions = bucket.dig("spec", "forProvider", "permissions") || []
+      permission = permissions.find { |candidate| candidate["role"] == "reader" } || permissions.first
+      permission&.dig("userRefs", 0, "name")
+    end
+
+    # Fetched eagerly (even for re-deploys that copy no data) so no origin-side nctl calls remain after inventory.
+    def bucketuser_credentials(user)
+      return [nil, nil] unless user
+
+      %w[--print-access-key --print-secret-key].map do |flag|
+        stdout, = system!("nctl get bucketuser #{user} -p #{project} #{flag}", quiet: true)
+        stdout.strip
+      end
+    end
+
+    def inventorize_key_value_store
+      key_value_store = origin_resource("KeyValueStore")
+      self.origin_key_value_store_name = key_value_store&.dig("metadata", "name")
+      self.origin_key_value_store_location = key_value_store&.dig("spec", "forProvider", "location")
+    end
+
+    def inventorize_bucket
+      bucket = origin_resource("Bucket")
+      self.origin_bucket_name = bucket&.dig("metadata", "name")
+      self.origin_bucket_endpoint = bucket&.dig("status", "atProvider", "endpoint")
+      self.origin_bucket_user = bucket_user(bucket)
+      self.origin_bucket_access_key, self.origin_bucket_secret_key = bucketuser_credentials(origin_bucket_user)
+    end
+
+    def ensure_required_tools
+      missing = required_tools.reject { |tool| installed?(tool) }
+      abort "missing #{missing.join(", ")} to copy the origin data, install them and rerun" if missing.any?
+    end
+
+    def required_tools
+      tools = []
+      tools += %w[aws pg_restore psql zstd] if origin_backup_path
+      tools += %w[aws] if origin_bucket_name
+      tools.uniq
+    end
+
+    def installed?(tool)
+      ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).any? { |dir| File.executable?(File.join(dir, tool)) }
+    end
+
+    def resources
+      # nctl hides the whole listing behind its no-resources banner when the project holds
+      # no nine-owned resources; the plain listing still contains everything else.
+      @resources ||= nctl_resources("nctl get all -p #{project} --include-nine-resources -o yaml").presence ||
+                     nctl_resources("nctl get all -p #{project} -o yaml")
+    end
+
+    def origin_resources
+      @origin_resources ||= resources.select do |resource|
+        resource.dig("metadata", "labels", "nine.ch/owned-by") != "nine" &&
+          resource.dig("metadata", "name").include?(origin_pattern)
+      end
+    end
+
+    def origin_app_name!
+      abort "project '#{project}' holds no resources, is its name right?" if resources.empty?
+
+      origin_resource("Application")&.dig("metadata", "name") ||
+        abort("no origin app (name containing '#{origin_pattern}') found in #{project}")
+    end
+
+    def origin_resource(kind)
+      origin_resources.find { |resource| resource["kind"] == kind }
+    end
+
+    def resources_of(kind)
+      resources.select { |resource| resource["kind"] == kind }
+    end
+
+    def backup_bucket_name
+      return nil unless origin_database_name
+
+      resources_of("DatabaseBackupSchedule")
+        .find { |schedule| schedule.dig("spec", "forProvider", "source", "name") == origin_database_name }
+        &.dig("status", "atProvider", "targetBucket", "name")
+    end
+
+    def latest_backup_path
+      return nil unless origin_backup_bucket
+
+      resources_of("DatabaseBackup")
+        .select { |backup| backup.dig("spec", "forProvider", "bucket", "name") == origin_backup_bucket }
+        .select { |backup| backup.dig("status", "atProvider", "state") == "succeeded" }
+        .max_by { |backup| backup.dig("status", "atProvider", "end").to_s }
+        &.dig("status", "atProvider", "path")
+    end
+
+    def bucket_resource(name)
+      name && resources_of("Bucket").find { |bucket| bucket.dig("metadata", "name") == name }
+    end
+  end
+end

data/lib/renuo/cli/services/deploio/start_target_app.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "shellwords"
+
+class Renuo::Cli::Services::Deploio
+  class StartTargetApp < Actor
+    include ReviewAppHelper
+
+    input :project
+    input :target_app_name, must: satisfy(SafeReviewAppNameContract)
+    input :branch
+
+    def call
+      puts "starting app #{target_app_name}..."
+      system! "nctl update app #{target_app_name} -p #{project} --git-revision #{Shellwords.escape(branch)} " \
+              "--skip-repo-access-check --no-pause"
+    end
+  end
+end

data/lib/renuo/cli/services/deploio/update_target_app_env.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "shellwords"
+require_relative "create_s3"
+
+class Renuo::Cli::Services::Deploio
+  class UpdateTargetAppEnv < Actor
+    include ReviewAppHelper
+
+    SENSITIVE_ENV_KEYS = (%w[DATABASE_URL REDIS_URL] + CreateS3::SENSITIVE_ENV_KEYS).freeze
+
+    input :project
+    input :target_app_name, must: satisfy(SafeReviewAppNameContract)
+    input :envs
+
+    def call
+      return if env.empty?
+
+      puts "updating env of app #{target_app_name}..."
+      system! ["nctl update app #{target_app_name} -p #{project}",
+               env_flag("--env", env.except(*SENSITIVE_ENV_KEYS)),
+               env_flag("--sensitive-env", env.slice(*SENSITIVE_ENV_KEYS))].compact.join(" ")
+    end
+
+    private
+
+    def env
+      @env ||= envs.reduce({}, :merge)
+    end
+
+    def env_flag(flag, variables)
+      return if variables.empty?
+
+      "#{flag} #{Shellwords.escape(variables.map { |key, value| "#{key}=#{value}" }.join(";"))}"
+    end
+  end
+end

data/lib/renuo/cli/services/deploio.rb

@@ -3,7 +3,21 @@
 require "shellwords"
 
 class Renuo::Cli::Services::Deploio
+  extend ReviewAppHelper
+
   class << self
+    # A named nctl get that misses searches every project of the organization,
+    # which project-scoped service accounts are forbidden to do.
+    def resource_exists?(type, name, project)
+      nctl_resources("nctl get #{type} -p #{project} -o yaml").any? do |resource|
+        resource.dig("metadata", "name") == name
+      end
+    end
+
+    def fetch_resource(type, name, project)
+      nctl_resources("nctl get #{type} #{name} -p #{project} -o yaml").first
+    end
+
     def fetch_apps
       active_org, available_orgs = fetch_whoami_orgs
 
@@ -33,9 +47,7 @@ class Renuo::Cli::Services::Deploio
     end
 
     def fetch_vms
-      fetch_vms_cmd = %(nctl get cloudvm -A -o yaml)
-      stdout, stderr, status = Open3.capture3 fetch_vms_cmd
-      raise "Error fetching Deploio app list: #{stderr}" unless status.success?
+      stdout, = run_nctl_command(%(nctl get cloudvm -A -o yaml))
 
       YAML.load_stream(stdout).map do |app|
         {
@@ -61,7 +73,7 @@ class Renuo::Cli::Services::Deploio
     end
 
     def run_nctl_command(cmd)
-      stdout, stderr, status = Open3.capture3(cmd)
+      stdout, stderr, status = system!(cmd, quiet: true, abort_on_failure: false)
       raise "Error fetching Deploio app list: #{stderr}" unless status.success?
 
       [stdout, stderr, status]

data/lib/renuo/cli/templates/semaphore/review-app-deploy.yml.erb

@@ -0,0 +1,44 @@
+version: v1.0
+name: review-app-deploy
+
+agent:
+  machine:
+    type: e1-standard-2
+    os_image: ubuntu2404
+
+blocks:
+  - name: review-app-deploy
+    execution_time_limit:
+      minutes: 30
+    task:
+      secrets:
+        - name: <%= secret_name %>
+      jobs:
+        - name: deploying
+          commands:
+            - gem i -q renuo-cli
+            - renuo ci setup
+            # pg_restore rejects backups dumped by a newer PostgreSQL, so match or exceed the origin database's version.
+            - sudo apt-get install -qq postgresql-common
+            - sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
+            - sudo apt-get install -qq postgresql-client-18
+            - export PATH=/usr/lib/postgresql/18/bin:$PATH
+            - renuo review-app:deploy --project <%= project %> --branch $SEMAPHORE_GIT_WORKING_BRANCH --origin-pattern <%= origin_pattern %>
+            # Make sure your review apps have the right environment variables. We don't copy from origin.
+            #
+            # -| nctl update app $(renuo review-app:name --branch $SEMAPHORE_GIT_WORKING_BRANCH) --project <%= project %> \
+            #      --env "KEY=value" --sensitive-env "SECRET=$FROM_SEMAPHORE_SECRET" \
+            #      --build-env "KEY=value" --sensitive-build-env "KEY=value"
+            #
+            # To find out which env variables a previously existing app uses, inspect its spec:
+            #
+            # - nctl get app existing-app --project <%= project %> -o yaml
+            #
+            # The runtime env is at .spec.forProvider.config.env, the build env at .spec.forProvider.buildEnv.
+            #
+            # If the origin's env variables are safe to reuse on review apps, you can replace this whole
+            # block (deploy, env setup, and start) with a single command:
+            #
+            # - renuo review-app:deploy --project <%= project %> --branch $SEMAPHORE_GIT_WORKING_BRANCH --origin-pattern <%= origin_pattern %> --keep-envs=build,sensitive-build,runtime,sensitive-runtime --start
+            - renuo review-app:start --project <%= project %> --branch $SEMAPHORE_GIT_WORKING_BRANCH
+            - renuo ci check-deploio-status $(renuo review-app:name --branch $SEMAPHORE_GIT_WORKING_BRANCH) --project <%= project %>

data/lib/renuo/cli/templates/semaphore/review-app-purge.yml.erb

@@ -0,0 +1,21 @@
+version: v1.0
+name: review-app-purge
+
+agent:
+  machine:
+    type: e1-standard-2
+    os_image: ubuntu2404
+
+blocks:
+  - name: review-app-purge
+    execution_time_limit:
+      minutes: 15
+    task:
+      secrets:
+        - name: <%= secret_name %>
+      jobs:
+        - name: purging
+          commands:
+            - gem i -q renuo-cli
+            - renuo ci setup
+            - renuo review-app:purge --project <%= project %> --branch $SEMAPHORE_GIT_WORKING_BRANCH

data/lib/renuo/cli/templates/semaphore/semaphore.yml.erb

@@ -24,6 +24,7 @@ global_job_config:
       - source .semaphore/bin/cache_restore rails
       - bundle config set deployment 'true'
       - bundle config set path 'vendor/bundle'
+      - bundle config set clean 'true'
       - bundle install --retry 3
 
 blocks:

data/lib/renuo/cli/version.rb

@@ -3,7 +3,7 @@
 # simplecov:disable
 module Renuo
   class Cli
-    VERSION = "4.26.0"
+    VERSION = "4.28.0"
     NAME = "renuo-cli"
   end
 end

data/lib/renuo/cli.rb

@@ -10,11 +10,21 @@ require "net/http"
 require "open3"
 require "psych"
 require "securerandom"
+require "service_actor"
 require "tmpdir"
 require "uri"
 require "renuo/cli/version"
 
+module Renuo
+  class Cli
+    module Commands; end
+    module Services; end
+  end
+end
+
 Dir[File.expand_path("cli/helpers/*.rb", __dir__)].each { |f| require f }
+require_relative "cli/services/deploio"
+Dir[File.expand_path("cli/services/deploio/*.rb", __dir__)].each { |f| require f }
 
 def agree(question)
   question += " [y/n]" if question