deploio-cli 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eae3786b2b0c98a4d02b1ac9037e9536da6cfc90ab7120ce80d4e98a76df90d7
-  data.tar.gz: ac86804058048327a006cd3343839299408425dd4e3702fe99aafc571e8e372c
+  metadata.gz: 1ef7a08d44a816dd88331bf7e007a536ef399b10afbf05229bef8e1e3137629e
+  data.tar.gz: d042f89dc44e64df49d599f1291f88151af45461132156af5c36486fa6408baa
 SHA512:
-  metadata.gz: 9a6a68ed578e07a9d0026f1dd61f3971ddd5f5e6b2ec590e2960644bef88de65cf054c04c2255051e0b550061eaa965a521b5700bfaf139ae5e3d76bf1f1aa19
-  data.tar.gz: 36614043d839a6a59c7ecc761f6d47ce8b9ff05cf149a630f33b17cb10b45ed8e374c2d1293c3ef7b9d98324290eeeb9332c2c530aea90642e18878faa08b9c4
+  metadata.gz: 603238b92b6b23a32bf92f81a0a89a4cc0edbf8daad40519f1f165c095629ccb5f83775f01f0aae67541f6177f845ebc9797584fdcdb977425fa2307c2d8f98e
+  data.tar.gz: 2ac16607435cc562bcf3d3e94066002315bb06f198122fe6d088ee25782f9153106ab065e217f34a86d5aa99a4cb2d7d154876eabcc958266f2e74cc6771ef09

data/README.md

@@ -6,6 +6,7 @@ A CLI for [Deploio](https://www.deplo.io/) that wraps [`nctl`](https://github.co
 
 - Ruby 3.3+
 - nctl version 1.10.0 or higher
+- [rclone](https://rclone.org/) (`brew install rclone`) — only needed for `deploio pg backups` on economy-tier databases
 
 ## Installation
 
@@ -18,7 +19,7 @@ bundle install
 bundle exec bin/deploio --help
 ```
 
-### As a gem (coming soon)
+### As a gem
 
 ```bash
 gem install deploio-cli
@@ -90,6 +91,13 @@ SERVICES
   deploio services -p PROJECT --connected-apps  Show which apps use each service (requires -p)
   deploio services --chf                    Show estimated monthly price (CHF) for each service
 
+POSTGRESQL
+  deploio pg                                  List all PostgreSQL databases
+  deploio pg:info NAME                        Show database details
+  deploio pg backups list NAME                List available backups
+  deploio pg backups download NAME            Download the latest backup
+  deploio pg backups capture NAME             Capture a new backup
+
 LOGS
   deploio logs -a APP             Show recent logs
   deploio logs -a APP --tail      Stream logs continuously

data/lib/deploio/cli.rb

@@ -100,6 +100,8 @@ module Deploio
 
     private
 
+    # The aliases above hand off with Thor.start, which starts from a fresh
+    # option parse, so the shared class options have to be passed as arguments.
     def build_option_args
       args = []
       args << "--dry-run" if options[:dry_run]

data/lib/deploio/commands/postgresql_backups.rb

@@ -1,3 +1,5 @@
+require "time"
+
 module Deploio
   module Commands
     class PostgreSQLBackups < Thor
@@ -5,70 +7,89 @@ module Deploio
 
       namespace "pg:backups"
 
+      DEDICATED_KIND = "Postgres"
+      ECONOMY_KIND = "PostgresDatabase"
+
       desc "capture NAME", "Capture a new backup for the specified PostgreSQL database"
       def capture(name)
-        setup_options
-        resolver = PgDatabaseResolver.new(nctl_client: @nctl)
-        db_ref = resolver.resolve(database_name: name)
-        data = @nctl.get_pg_database(db_ref)
-        kind = data["kind"] || ""
+        backup_service_for(name).capture
+      rescue Deploio::Error => e
+        Output.error(e.message)
+        exit 1
+      end
 
-        unless kind == "Postgres" || @nctl.dry_run
-          Output.error("Backups can only be captured for PostgreSQL databases. (shared dbs are not supported)")
-          exit 1
+      desc "list NAME", "List the available backups for the specified PostgreSQL database"
+      def list(name)
+        backups = backup_service_for(name).backups
+        if backups.empty?
+          Output.warning("No backups found for '#{name}'")
+          return
         end
 
-        fqdn = data.dig("status", "atProvider", "fqdn")
-        if fqdn.nil? || fqdn.empty?
-          Output.error("Database FQDN not found; cannot capture backup.")
-          exit 1
+        rows = backups.map do |backup|
+          [format_time(backup["ModTime"]), format_size(backup["Size"]), backup["Name"]]
         end
-
-        cmd = ["ssh", "dbadmin@#{fqdn}", "sudo nine-postgresql-backup"]
-        Output.command(cmd.join(" "))
-        system(*cmd) unless @nctl.dry_run
+        Output.table(rows, headers: ["DATE", "SIZE", "NAME"])
+      rescue Deploio::Error => e
+        Output.error(e.message)
+        exit 1
       end
 
       desc "download NAME [--output destination_path]", "Download the latest backup for the specified PostgreSQL database instance"
       method_option :output, type: :string, desc: "Output file path (defaults to current directory with auto-generated name)"
       method_option :db_name, type: :string, desc: "If there are multiple DBs, specify which one to download the backup for", default: nil
       def download(name)
-        destination = options[:output] || "./#{name}-latest-backup.zst"
+        service = backup_service_for(name)
+        destination = merged_options[:output] || service.default_destination
+        backup = service.download(destination: destination, db_name: merged_options[:db_name])
 
+        # Only the economy tier knows when its backup was taken.
+        if backup
+          Output.success("Downloaded backup from #{format_time(backup["ModTime"])} to #{destination}")
+        else
+          Output.success("Downloaded backup to #{destination}")
+        end
+      rescue Deploio::Error => e
+        Output.error(e.message)
+        exit 1
+      end
+
+      private
+
+      def backup_service_for(name)
         setup_options
         resolver = PgDatabaseResolver.new(nctl_client: @nctl)
         db_ref = resolver.resolve(database_name: name)
         data = @nctl.get_pg_database(db_ref)
-        kind = data["kind"] || ""
-
-        unless kind == "Postgres" || @nctl.dry_run
-          Output.error("Backups can only be downloaded for PostgreSQL databases. (shared dbs are not supported)")
-          exit 1
-        end
+        raise Deploio::Error, "Could not read database '#{db_ref.full_name}'." if data.nil?
 
-        databases = data.dig("status", "atProvider", "databases")&.keys || []
-        databases.reject! { |db| db.strip.empty? }
-        if databases.empty?
-          Output.error("No databases found in PostgreSQL instance; cannot download backup.")
-          exit 1
-        elsif databases.size > 1 && options[:db_name].nil?
-          Output.error("Multiple databases found in PostgreSQL instance")
-          Output.error("Databases: #{databases.join(", ")}")
-          Output.error("Please specify the database name using the --db_name option.")
-          exit 1
+        case data["kind"]
+        when DEDICATED_KIND
+          PostgresBackupService.new(data: data, name: name, dry_run: @nctl.dry_run)
+        when ECONOMY_KIND
+          PostgresDatabaseBackupService.new(db_ref: db_ref, data: data, nctl_client: @nctl, name: name)
+        else
+          raise Deploio::UnsupportedBackupOperationError,
+            "Backups are not supported for databases of kind '#{data["kind"]}'."
         end
+      end
 
-        db_name = options[:db_name] || databases.first
+      def format_time(value)
+        Time.parse(value.to_s).localtime.strftime("%Y-%m-%d %H:%M")
+      rescue ArgumentError, TypeError
+        value.to_s
+      end
 
-        fqdn = data.dig("status", "atProvider", "fqdn")
-        if fqdn.nil? || fqdn.empty?
-          Output.error("Database FQDN not found; cannot download backup.")
-          exit 1
+      def format_size(bytes)
+        bytes = bytes.to_i
+        units = ["B", "KiB", "MiB", "GiB", "TiB"]
+        index = 0
+        size = bytes.to_f
+        while size >= 1024 && index < units.size - 1
+          size /= 1024
+          index += 1
         end
-
-        cmd = ["rsync", "-avz", "dbadmin@#{fqdn}:~/backup/postgresql/latest/customer/#{db_name}/#{db_name}.zst", destination]
-        Output.command(cmd.join(" "))
-        system(*cmd) unless @nctl.dry_run
+        (index.zero? ? "#{bytes} B" : format("%.1f %s", size, units[index]))
       end
     end
   end

data/lib/deploio/completion_generator.rb

@@ -61,6 +61,7 @@ module Deploio
         "orgs:set" => "'1:organization:_#{program_name}_orgs_list'",
         "pg:info" => "'1:database:_#{program_name}_pg_databases_list'",
         "pg:backups:capture" => "'1:database:_#{program_name}_pg_databases_list'",
+        "pg:backups:list" => "'1:database:_#{program_name}_pg_databases_list'",
         "pg:backups:download" => "'1:database:_#{program_name}_pg_databases_list'"
       }
     end

data/lib/deploio/nctl_client.rb

@@ -197,6 +197,14 @@ module Deploio
       nil
     end
 
+    def get_bucket_user_access_key(name, project:)
+      capture("get", "bucketuser", name, "--project", project, "--print-access-key").strip
+    end
+
+    def get_bucket_user_secret_key(name, project:)
+      capture("get", "bucketuser", name, "--project", project, "--print-secret-key").strip
+    end
+
     def get_projects
       output = capture("get", "projects", "-o", "json")
       return [] if output.nil? || output.empty?

data/lib/deploio/postgres_backup_service.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Deploio
+  # Backups for the dedicated tier (kind: Postgres), where we own the whole
+  # database server and reach it over SSH
+  # The naming is confusing, but this is how Nine names them and how the resources appear, so prefer to stay
+  # consistent with that
+  class PostgresBackupService
+    DEFAULT_EXTENSION = ".zst"
+
+    # @param name [String] the name the user typed, used for hints in messages
+    def initialize(data:, name: nil, dry_run: false)
+      @data = data || {}
+      @name = name
+      @dry_run = dry_run
+    end
+
+    def default_destination = "./#{@name}-latest-backup#{DEFAULT_EXTENSION}"
+
+    def backups
+      raise Deploio::UnsupportedBackupOperationError,
+        "Listing backups is not yet supported for dedicated PostgreSQL instances; Feel free to implement it!\n" \
+        "Use 'deploio pg backups download #{@name}' to fetch it."
+    end
+
+    def capture
+      cmd = ["ssh", "dbadmin@#{fqdn}", "sudo nine-postgresql-backup"]
+      Output.command(cmd.join(" "))
+      system(*cmd) unless @dry_run
+    end
+
+    def download(destination:, db_name: nil)
+      name = resolve_db_name(db_name)
+
+      cmd = ["rsync", "-av", "dbadmin@#{fqdn}:~/backup/postgresql/latest/customer/#{name}/#{name}.zst", destination]
+      Output.command(cmd.join(" "))
+      system(*cmd) unless @dry_run
+
+      nil
+    end
+
+    private
+
+    def resolve_db_name(db_name)
+      if databases.empty?
+        raise Deploio::Error, "No databases found in PostgreSQL instance; cannot download backup."
+      elsif databases.size > 1 && db_name.nil?
+        raise Deploio::Error,
+          "Multiple databases found in PostgreSQL instance\n" \
+          "Databases: #{databases.join(", ")}\n" \
+          "Please specify the database name using the --db_name option."
+      end
+
+      db_name || databases.first
+    end
+
+    def databases
+      @databases ||= (@data.dig("status", "atProvider", "databases")&.keys || []).reject { |db| db.strip.empty? }
+    end
+
+    def fqdn
+      value = @data.dig("status", "atProvider", "fqdn")
+      raise Deploio::Error, "Database FQDN not found; cannot reach the database server." if value.nil? || value.empty?
+
+      value
+    end
+  end
+end

data/lib/deploio/postgres_database_backup_service.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Deploio
+  # Backups for the economy tier (kind: PostgresDatabase), where the database
+  # lives on a shared server we have no access to.
+  # The naming is confusing, but this is how Nine names them and how the resources appear, so prefer to stay
+  # consistent with that
+  class PostgresDatabaseBackupService
+    DEFAULT_EXTENSION = ".sql.zst"
+
+    BACKUP_SCHEDULE_LABEL = "DatabaseBackupSchedule"
+
+    def initialize(db_ref:, data:, nctl_client:, name: nil, rclone_client_factory: nil)
+      @db_ref = db_ref
+      @data = data || {}
+      @nctl = nctl_client
+      @name = name || db_ref.full_name
+      @rclone_client_factory = rclone_client_factory || method(:build_rclone_client)
+    end
+
+    def default_destination
+      "./#{@name}-latest-backup#{DEFAULT_EXTENSION}"
+    end
+
+    # Nine takes these backups on a schedule
+    # (you can even see them using `kubectl get databasebackupschedules.storage.nine.ch -n renuo-chess-tracker -o json`)
+    # => there is no way to trigger one.
+    def capture
+      raise Deploio::UnsupportedBackupOperationError,
+        "'#{@db_ref.full_name}' is an economy-tier database. Those are backed up automatically " \
+        "on their configured schedule and cannot be captured manually.\n" \
+        "Use 'deploio pg backups list #{@name}' to see the available backups."
+    end
+
+    def backups
+      @backups ||= begin
+        entries = rclone.list(bucket_name)
+        entries = entries.select { |e| own_backup?(e["Name"].to_s) }
+        entries.sort_by { |e| e["ModTime"].to_s }.reverse
+      end
+    end
+
+    def download(destination:, db_name: nil)
+      backup = backups.first
+      unless backup
+        raise Deploio::Error, "No backups found for '#{@db_ref.full_name}' in bucket '#{bucket_name}'."
+      end
+
+      rclone.download(bucket_name, backup["Name"], destination)
+      backup
+    end
+
+    private
+
+    def own_backup?(object_name)
+      return true if instance_name.empty?
+
+      object_name.start_with?("PostgresDatabase-#{instance_name}-")
+    end
+
+    def instance_name
+      @instance_name ||= @data.dig("status", "atProvider", "name").to_s
+    end
+
+    def bucket_name
+      bucket.dig("metadata", "name")
+    end
+
+    # The PostgresDatabase resource holds no reference to its backup bucket
+    # Changes with service connections (I assume), but we're not there yet as we still have projects with the old setup.
+    # Therefore, we match by name
+    def bucket
+      @bucket ||= begin
+        candidates = @nctl.get_services_by_type("bucket", project: @db_ref.project_name).select do |bucket|
+          backup_bucket_for_database?(bucket)
+        end
+
+        if candidates.empty?
+          raise Deploio::Error,
+            "No backup bucket found for '#{@db_ref.full_name}'. " \
+            "Check that backups are enabled for this database (spec.forProvider.backupSchedule)."
+        end
+
+        candidates.first
+      end
+    end
+
+    def backup_bucket_for_database?(bucket)
+      metadata = bucket["metadata"] || {}
+      labels = metadata["labels"] || {}
+      return false unless labels["nine.ch/controllerKind"] == BACKUP_SCHEDULE_LABEL
+
+      metadata["name"].to_s.match?(/\Apostgresdatabase-#{Regexp.escape(@db_ref.database_name)}-[0-9a-f]{7}\z/)
+    end
+
+    def rclone
+      @rclone ||= @rclone_client_factory.call
+    end
+
+    def build_rclone_client
+      endpoint = bucket.dig("status", "atProvider", "endpoint")
+      if endpoint.nil? || endpoint.to_s.empty?
+        raise Deploio::Error, "Backup bucket '#{bucket_name}' has no endpoint; cannot access backups."
+      end
+
+      client = RcloneClient.new(
+        endpoint: "https://#{endpoint}",
+        access_key: @nctl.get_bucket_user_access_key(bucket_name, project: @db_ref.project_name),
+        secret_key: @nctl.get_bucket_user_secret_key(bucket_name, project: @db_ref.project_name),
+        dry_run: @nctl.dry_run
+      )
+      client.check_requirements unless @nctl.dry_run
+      client
+    end
+  end
+end

data/lib/deploio/rclone_client.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "json"
+require "open3"
+
+module Deploio
+  class RcloneClient
+    REMOTE = "DEPLOIO"
+
+    attr_reader :dry_run
+
+    def initialize(endpoint:, access_key:, secret_key:, dry_run: false)
+      @endpoint = endpoint
+      @access_key = access_key
+      @secret_key = secret_key
+      @dry_run = dry_run
+    end
+
+    def check_requirements
+      check_rclone_installed
+    end
+
+    def list(bucket)
+      output = capture("lsjson", remote_path(bucket))
+      return [] if output.nil? || output.empty?
+
+      data = JSON.parse(output)
+      data.is_a?(Array) ? data : []
+    rescue JSON::ParserError
+      []
+    end
+
+    def download(bucket, object, destination)
+      run("copyto", remote_path(bucket, object), destination, "--progress", "--stats-one-line")
+    end
+
+    private
+
+    # --s3-no-check-bucket skips the HeadBucket call, which the read-only bucket
+    # user is not permitted to make.
+    # See also https://docs.nine.ch/docs/object-storage/object-storage-client-tools#rclone
+    def build_command(args)
+      ["rclone", *args.map(&:to_s), "--s3-no-check-bucket"]
+    end
+
+    def remote_path(bucket, object = nil)
+      object ? "#{REMOTE}:#{bucket}/#{object}" : "#{REMOTE}:#{bucket}"
+    end
+
+    def env
+      {
+        "RCLONE_CONFIG_#{REMOTE}_TYPE" => "s3",
+        "RCLONE_CONFIG_#{REMOTE}_PROVIDER" => "Other",
+        "RCLONE_CONFIG_#{REMOTE}_ENDPOINT" => @endpoint,
+        "RCLONE_CONFIG_#{REMOTE}_ACCESS_KEY_ID" => @access_key,
+        "RCLONE_CONFIG_#{REMOTE}_SECRET_ACCESS_KEY" => @secret_key
+      }
+    end
+
+    def capture(*args)
+      cmd = build_command(args)
+      if dry_run
+        Output.command(cmd.join(" "))
+        return ""
+      end
+
+      puts "> #{cmd.join(" ")}" if ENV["DEPLOIO_DEBUG"]
+      stdout, stderr, status = Open3.capture3(env, *cmd)
+      unless status.success?
+        raise Deploio::RcloneError, "rclone command failed: #{stderr}"
+      end
+
+      stdout
+    end
+
+    def run(*args)
+      cmd = build_command(args)
+      Output.command(cmd.join(" "))
+      return true if dry_run
+
+      unless system(env, *cmd)
+        raise Deploio::RcloneError, "rclone command failed: #{cmd.join(" ")}"
+      end
+
+      true
+    end
+
+    def check_rclone_installed
+      _stdout, _stderr, status = Open3.capture3("rclone", "version")
+      return if status.success?
+
+      raise Deploio::RcloneError,
+        "rclone not found. Please install it: brew install rclone"
+    rescue Errno::ENOENT
+      raise Deploio::RcloneError,
+        "rclone not found. Please install it: brew install rclone"
+    end
+  end
+end

data/lib/deploio/shared_options.rb

@@ -7,8 +7,8 @@ module Deploio
     def self.included(base)
       base.class_option :app, aliases: "-a", type: :string, desc: "App in <project>-<app> format"
       base.class_option :org, aliases: "-o", type: :string, desc: "Organization"
-      base.class_option :dry_run, type: :boolean, default: false, desc: "Print commands without executing"
-      base.class_option :no_color, type: :boolean, default: false, desc: "Disable colored output"
+      base.class_option :dry_run, type: :boolean, desc: "Print commands without executing"
+      base.class_option :no_color, type: :boolean, desc: "Disable colored output"
 
       base.define_singleton_method(:exit_on_failure?) { true }
     end

data/lib/deploio/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deploio
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/deploio.rb

@@ -8,8 +8,11 @@ require_relative "deploio/output"
 require_relative "deploio/app_ref"
 require_relative "deploio/pg_database_ref"
 require_relative "deploio/nctl_client"
+require_relative "deploio/rclone_client"
 require_relative "deploio/app_resolver"
 require_relative "deploio/pg_database_resolver"
+require_relative "deploio/postgres_backup_service"
+require_relative "deploio/postgres_database_backup_service"
 require_relative "deploio/price_fetcher"
 require_relative "deploio/shared_options"
 require_relative "deploio/cli"
@@ -19,4 +22,6 @@ module Deploio
   class AppNotFoundError < Error; end
   class PgDatabaseNotFoundError < Error; end
   class NctlError < Error; end
+  class RcloneError < Error; end
+  class UnsupportedBackupOperationError < Error; end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deploio-cli
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Renuo AG
@@ -67,7 +67,10 @@ files:
 - lib/deploio/output.rb
 - lib/deploio/pg_database_ref.rb
 - lib/deploio/pg_database_resolver.rb
+- lib/deploio/postgres_backup_service.rb
+- lib/deploio/postgres_database_backup_service.rb
 - lib/deploio/price_fetcher.rb
+- lib/deploio/rclone_client.rb
 - lib/deploio/shared_options.rb
 - lib/deploio/templates/completion.zsh.erb
 - lib/deploio/utils.rb