kamal-backup 0.5.2 → 1.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 341913561478a45fc1dd88b376daae6977a180eba43739963c6e9055d5c064ab
-  data.tar.gz: 69a572c55cc9b33fcd0ab0a3918111cf7c3e26f3519d01de855c2fa73de08b2d
+  metadata.gz: 8e351aa7e872cd48344295084005c0cdb418169affa209b6f6c3b704372e96d5
+  data.tar.gz: ee83efba870cf740e9d95898e62191611ddaa8153711d8030c90045166d224e3
 SHA512:
-  metadata.gz: af86080b3f6524990cd0df83bed996370e4cff186aaea4338e8ab9e9ddfb595313d58b96dd4f9fc0a17b758ff8869a7fd836840370bfe76b94dcad5d195b7455
-  data.tar.gz: 257939e10db7512ea4f12d0bd23d8ba472a41c6987ef974cc69c978653491b0e4b107a7e312fb34431745a5b855dae6919a58e8cdd626e8277c9276030a5a6d6
+  metadata.gz: 739282c586403339edda3f475190355400f639c0c1eba0b8145531f964a8be6815689f6252a8d4d3f9fb7db42efd3413f1188ff0243503d15d27379bf32c4998
+  data.tar.gz: 6abaadf21b62c4d106b92fbce3c0f703d4bb42d03cc6e86c47f9eb90e24942beda433a98c6f05acfb3557da3323ff551a85ad7b63fe6a0db484a06a3813f650f

data/README.md

@@ -125,7 +125,7 @@ bundle exec kamal-backup evidence
 
 - **Scheduled backups:** the accessory runs continuously and backs up on `backup.schedule`.
 - **Database and Active Storage coverage:** database dumps plus file-backed Active Storage files from mounted volumes.
-- **Restic underneath:** encrypted, deduplicated snapshots in S3-compatible storage, over SFTP, in a restic REST server, or in a filesystem repository.
+- **Restic underneath:** encrypted, deduplicated snapshots in native restic backends or any rclone remote; the accessory includes both rclone and an SSH client.
 - **Local restores:** inspect production data safely in your local Rails app.
 - **Restore drills:** restore into scratch production-side targets, run verification commands, and record the result.
 - **Security review evidence:** `kamal-backup evidence` prints redacted JSON with latest snapshots, `kamal-backup check` results, drills, retention, and tool versions.

data/lib/kamal_backup/databases/mysql.rb

@@ -20,6 +20,8 @@ module KamalBackup
           dump_binary,
           '--single-transaction',
           '--quick',
+          '--skip-comments',
+          '--no-tablespaces',
           '--routines',
           '--triggers',
           '--events'
@@ -28,6 +30,17 @@ module KamalBackup
         CommandSpec.new(argv: argv, env: password_env(connection))
       end
 
+      def restore_to_current(restic, snapshot, filename)
+        reset_database(current_connection)
+        super
+      end
+
+      def restore_to_scratch(restic, snapshot, filename, target:)
+        validate_scratch_restore_target(target)
+        reset_database(current_connection.merge(database: target))
+        restic.pipe_dump_to_command(snapshot, filename, scratch_restore_command(target))
+      end
+
       def current_restore_command
         connection = current_connection
         argv = [client_binary] + connection_args(connection)
@@ -53,6 +66,68 @@ module KamalBackup
 
       private
 
+      DATABASE_OBJECTS_SQL = <<~SQL
+        SELECT 'VIEW', HEX(TABLE_NAME)
+          FROM information_schema.VIEWS
+         WHERE TABLE_SCHEMA = DATABASE()
+        UNION ALL
+        SELECT CASE WHEN TABLE_TYPE = 'SEQUENCE' THEN 'SEQUENCE' ELSE 'TABLE' END, HEX(TABLE_NAME)
+          FROM information_schema.TABLES
+         WHERE TABLE_SCHEMA = DATABASE() AND TABLE_TYPE <> 'VIEW'
+        UNION ALL
+        SELECT ROUTINE_TYPE, HEX(ROUTINE_NAME)
+          FROM information_schema.ROUTINES
+         WHERE ROUTINE_SCHEMA = DATABASE()
+        UNION ALL
+        SELECT 'EVENT', HEX(EVENT_NAME)
+          FROM information_schema.EVENTS
+         WHERE EVENT_SCHEMA = DATABASE()
+      SQL
+      private_constant :DATABASE_OBJECTS_SQL
+
+      def reset_database(connection)
+        objects = database_objects(connection)
+        statements = ['SET FOREIGN_KEY_CHECKS=0;']
+        %w[VIEW TABLE SEQUENCE PROCEDURE FUNCTION EVENT].each do |type|
+          objects.fetch(type, []).each do |name|
+            statements << "DROP #{type} IF EXISTS #{quote_identifier(name)};"
+          end
+        end
+
+        Command.capture(
+          CommandSpec.new(argv: client_argv(connection), env: password_env(connection)),
+          redactor: redactor,
+          input: statements.join("\n")
+        )
+      rescue CommandError => e
+        raise CommandError.new(
+          "failed to reset the MySQL database before restoring: #{e.message}",
+          command: e.command,
+          stderr: e.stderr
+        )
+      end
+
+      def database_objects(connection)
+        command = CommandSpec.new(
+          argv: client_argv(
+            connection,
+            options: ['--batch', '--skip-column-names', '--raw', '--execute', DATABASE_OBJECTS_SQL]
+          ),
+          env: password_env(connection)
+        )
+        output = Command.capture(command, redactor: redactor, log_output: false).stdout
+        output.lines.each_with_object(Hash.new { |hash, key| hash[key] = [] }) do |line, objects|
+          type, hex_name = line.chomp.split("\t", 2)
+          next if type.to_s.empty? || hex_name.to_s.empty?
+
+          objects[type] << [hex_name].pack('H*')
+        end
+      end
+
+      def quote_identifier(value)
+        "`#{value.to_s.gsub('`', '``')}`"
+      end
+
       def validate_scratch_restore_target(target)
         raise ConfigurationError, 'scratch database must differ from the current MySQL database' if current_connection.fetch(:database) == target
 
@@ -92,6 +167,9 @@ module KamalBackup
 
       def parse_url(url)
         uri = URI.parse(url)
+        supported_schemes = %w[mysql mysql2 mariadb]
+        raise ConfigurationError, 'DATABASE_URL must use mysql://, mysql2://, or mariadb://' unless supported_schemes.include?(uri.scheme)
+
         database = uri.path.to_s.sub(%r{\A/}, '')
         raise ConfigurationError, "database name is missing in #{uri.scheme} DATABASE_URL" if database.empty?
 
@@ -114,6 +192,10 @@ module KamalBackup
         args
       end
 
+      def client_argv(connection, options: [])
+        [client_binary] + connection_args(connection) + options + [connection.fetch(:database)]
+      end
+
       def password_env(connection)
         connection[:password] ? { 'MYSQL_PWD' => connection[:password] } : {}
       end

data/lib/kamal_backup/databases/postgres.rb

@@ -30,11 +30,12 @@ module KamalBackup
       end
 
       def dump_command
-        argv = %w[pg_dump --format=custom --no-owner --no-privileges]
-        CommandSpec.new(argv: argv, env: current_connection)
+        connection = current_connection
+        argv = [postgres_binary('pg_dump', connection)] + %w[--format=custom --no-owner --no-privileges]
+        CommandSpec.new(argv: argv, env: connection)
       end
 
-      # Replace the target schema before restoring.
+      # Replace the target's user schemas before restoring.
       #
       # pg_restore --clean emits DROP TABLE for each object in the dump, but
       # PostgreSQL refuses to drop a table another table still references, and
@@ -45,22 +46,31 @@ module KamalBackup
       # exists", the COPYs never running, and the foreign keys rejected because
       # the tables they point at are still empty.
       #
-      # Dropping the schema first sidesteps the ordering problem entirely. This
-      # only runs for restore-to-current, which already means "replace this
-      # database"; scratch restores target a separate database and are
-      # untouched.
+      # Dropping every non-system schema first sidesteps the ordering problem
+      # entirely and removes target-only custom schemas as well as public.
+      # Current and scratch restores both promise that the target represents
+      # the selected snapshot, so both paths reset their target schema.
       def restore_to_current(restic, snapshot, filename)
-        reset_current_schema
+        reset_schema(current_connection)
         result = super
         ensure_restore_reported_no_errors(result)
         result
       end
 
+      def restore_to_scratch(restic, snapshot, filename, target:)
+        validate_scratch_restore_target(target)
+        reset_schema(current_connection.merge('PGDATABASE' => target))
+        result = restic.pipe_dump_to_command(snapshot, filename, scratch_restore_command(target))
+        ensure_restore_reported_no_errors(result)
+        result
+      end
+
       def current_restore_command
         connection = current_connection
         database = connection.fetch('PGDATABASE')
 
-        argv = %w[pg_restore --clean --if-exists --no-owner --no-privileges --dbname]
+        argv = [postgres_binary('pg_restore', connection)] +
+               %w[--clean --if-exists --no-owner --no-privileges --exit-on-error --dbname]
         argv << database
         CommandSpec.new(argv: argv, env: connection)
       end
@@ -68,7 +78,8 @@ module KamalBackup
       def scratch_restore_command(target)
         connection = current_connection.merge('PGDATABASE' => target)
 
-        argv = %w[pg_restore --clean --if-exists --no-owner --no-privileges --dbname]
+        argv = [postgres_binary('pg_restore', connection)] +
+               %w[--clean --if-exists --no-owner --no-privileges --exit-on-error --dbname]
         argv << target
         CommandSpec.new(argv: argv, env: connection)
       end
@@ -100,20 +111,71 @@ module KamalBackup
         )
       end
 
-      def reset_current_schema
-        connection = current_connection
-        argv = ['psql', '--quiet', '--no-psqlrc', '--set', 'ON_ERROR_STOP=1', '--command', RESET_SCHEMA_SQL]
+      def reset_schema(connection)
+        argv = [postgres_binary('psql', connection), '--quiet', '--no-psqlrc', '--set', 'ON_ERROR_STOP=1',
+                '--command', RESET_SCHEMAS_SQL]
         Command.capture(CommandSpec.new(argv: argv, env: connection), redactor: redactor)
       rescue CommandError => e
         raise CommandError.new(
-          "failed to reset the public schema before restoring: #{e.message}",
+          "failed to reset PostgreSQL schemas before restoring: #{e.message}",
           command: e.command,
           stderr: e.stderr
         )
       end
 
-      RESET_SCHEMA_SQL = 'DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;'
-      private_constant :RESET_SCHEMA_SQL
+      RESET_SCHEMAS_SQL = <<~SQL
+        DO $$
+        DECLARE
+          schema_name text;
+        BEGIN
+          FOR schema_name IN
+            SELECT nspname
+              FROM pg_namespace
+             WHERE nspname !~ '^pg_'
+               AND nspname <> 'information_schema'
+          LOOP
+            EXECUTE format('DROP SCHEMA %I CASCADE', schema_name);
+          END LOOP;
+        END
+        $$;
+        CREATE SCHEMA public;
+      SQL
+      private_constant :RESET_SCHEMAS_SQL
+
+      def postgres_binary(name, connection)
+        root = value('KAMAL_BACKUP_POSTGRES_CLIENT_ROOT')
+        return name if root.to_s.empty?
+
+        major = postgres_server_major(connection)
+        binary = File.join(root, major.to_s, 'bin', name)
+        return binary if File.executable?(binary)
+
+        supported = Dir[File.join(root, '*', 'bin', name)].filter_map do |path|
+          path.split(File::SEPARATOR)[-3] if File.executable?(path)
+        end.sort
+        raise ConfigurationError,
+              "PostgreSQL #{major} is not supported by this accessory image; available client versions: #{supported.join(', ')}"
+      end
+
+      def postgres_server_major(connection)
+        @postgres_server_majors ||= {}
+        key = connection.values_at('PGHOST', 'PGPORT', 'PGUSER', 'PGDATABASE', 'PGSERVICE')
+        @postgres_server_majors[key] ||= begin
+          probe = value('KAMAL_BACKUP_POSTGRES_PROBE_BIN') || 'psql'
+          result = Command.capture(
+            CommandSpec.new(
+              argv: [probe, '--no-psqlrc', '--tuples-only', '--no-align', '--command', 'SHOW server_version_num'],
+              env: connection
+            ),
+            redactor: redactor,
+            log: false
+          )
+          version_number = Integer(result.stdout.strip, 10)
+          version_number / 10_000
+        rescue ArgumentError
+          raise ConfigurationError, 'could not determine the PostgreSQL server version'
+        end
+      end
 
       def validate_scratch_restore_target(target)
         raise ConfigurationError, 'scratch database must differ from the current PostgreSQL database' if current_connection.fetch('PGDATABASE') == target

data/lib/kamal_backup/databases/sqlite.rb

@@ -32,12 +32,12 @@ module KamalBackup
       end
 
       def restore_to_current(restic, snapshot, filename)
-        restic.write_dump_to_path(snapshot, filename, sqlite_source)
+        restore_database(restic, snapshot, filename, target: sqlite_source)
       end
 
       def restore_to_scratch(restic, snapshot, filename, target:)
         validate_scratch_restore_target(target)
-        restic.write_dump_to_path(snapshot, filename, target)
+        restore_database(restic, snapshot, filename, target: target)
       end
 
       def dump_command
@@ -67,6 +67,31 @@ module KamalBackup
       def sqlite_literal(value)
         "'#{value.to_s.gsub("'", "''")}'"
       end
+
+      def restore_database(restic, snapshot, filename, target:)
+        Tempfile.create(['kamal-backup-restore-', '.sqlite3']) do |tempfile|
+          tempfile.close
+          restic.write_dump_to_path(snapshot, filename, tempfile.path)
+          validate_database_file(tempfile.path)
+          FileUtils.mkdir_p(File.dirname(File.expand_path(target)))
+          Command.capture(
+            CommandSpec.new(argv: ['sqlite3', '-bail', target, ".restore #{sqlite_literal(tempfile.path)}"]),
+            redactor: redactor
+          )
+          validate_database_file(target)
+        end
+      end
+
+      def validate_database_file(path)
+        result = Command.capture(
+          CommandSpec.new(argv: ['sqlite3', '-bail', path, 'PRAGMA quick_check;']),
+          redactor: redactor,
+          log_output: false
+        )
+        return if result.stdout.strip == 'ok'
+
+        raise ConfigurationError, "SQLite integrity check failed for #{path}"
+      end
     end
   end
 end

data/lib/kamal_backup/evidence.rb

@@ -84,7 +84,8 @@ module KamalBackup
         mysql_dump: version_for(['mariadb-dump', '--version'], ['mysqldump', '--version']),
         mysql_client: version_for(['mariadb', '--version'], ['mysql', '--version']),
         sqlite3: version_for(['sqlite3', '--version']),
-        restic: version_for(%w[restic version])
+        restic: version_for(%w[restic version]),
+        rclone: version_for(%w[rclone version])
       }
     end
 

data/lib/kamal_backup/redactor.rb

@@ -2,8 +2,8 @@
 
 module KamalBackup
   class Redactor
-    SECRET_KEY_PATTERN = /(pass|password|secret|token|key|credential|authorization)/i
-    SENSITIVE_KEY_PATTERN = /(?:pass|password|secret|token|key|credential|authorization)|\A(?:user|username|pguser|.*_user|.*_username)\z/i
+    SECRET_KEY_PATTERN = /(?:pass|password|secret|token|key|credential|authorization)|(?:\A|_)pwd(?:\z|_)/i
+    SENSITIVE_KEY_PATTERN = /(?:pass|password|secret|token|key|credential|authorization)|(?:\A|_)pwd(?:\z|_)|\A(?:user|username|pguser|.*_user|.*_username)\z/i
     REDACTED = '[REDACTED]'
 
     def initialize(secret_values: [], env: ENV)

data/lib/kamal_backup/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KamalBackup
-  VERSION = '0.5.2'
+  VERSION = '1.0.0.rc1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kamal-backup
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 1.0.0.rc1
 platform: ruby
 authors:
 - crmne