litestream 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7942f70131f11cfcfad19c07442ab78571ad5bb28f67d8e38f1c35573a04413
-  data.tar.gz: 7584c2778dff02a7ded7078656ba57247146a67924c2ef05f4e1fe8407ea04e9
+  metadata.gz: 4b38d63aa3c23ba58c57dd91f178e069e898519f06b38e7ece5d0f9b9238f20d
+  data.tar.gz: 730e40b5ba2fd39a726902052333c049943adebbde252d2a769013585172d789
 SHA512:
-  metadata.gz: a48912bbcafb448473ebfd6ff378db5faa0a7f1830b786dff1b6574f8bd9ce210fd1f6b2f6652555544a38923a68fd87562abbca8e497db4282facc91a353ce0
-  data.tar.gz: a2a798e38ccf63d1fd21008bf1ef626a77652f8a8d1ef192fb1eeb11344f933d209f661dc3d24afd905d534ad5c295f4a0afb07b5cc45781ce152b2431746d72
+  metadata.gz: b07f6c0cd590a5ec095f778a1b473a6168adc5e747aadfbef2ce8dbea4047ef27ed9861ce0c36ac4d78bcdf7ea819a4e2d57db4c989ee8da9190523a77ae86cf
+  data.tar.gz: 0f9436ff88c9adf3b521c853e3a8e770b8aea4590fd2d8eb2cb575d48dfd7044a6156982b2e2924724c1bd5a5b97840eba567b143084abaf28786a4e50523135

data/README.md

@@ -165,6 +165,40 @@ You can forward arguments in whatever order you like, you simply need to ensure
     Disables environment variable expansion in configuration file.
 ```
 
+### Verification
+
+You can verify the integrity of your backed-up databases using the gem's provided `litestream:verify` rake task. This rake task requires that you specify which specific database you want to verify. As with the `litestream:restore` tasks, you pass arguments to the rake task via argument forwarding. For example, to verify the production database, you would run:
+
+```shell
+bin/rails litestream:verify -- --database=storage/production.sqlite3
+# or
+bundle exec rake litestream:verify -- --database=storage/production.sqlite3
+```
+
+The `litestream:verify` rake task takes the same options as the `litestream:restore` rake task. After restoring the backup, the rake task will verify the integrity of the restored database by ensuring that the restored database file
+
+1. exists,
+2. can be opened by SQLite, and
+3. sufficiently matches the original database file.
+
+Since point 3 is subjective, the rake task will output a message providing both the file size and number of tables of both the "original" and "restored" databases. You must manually verify that the restored database is within an acceptable range of the original database.
+
+The rake task will output a message similar to the following:
+
+```
+size
+  original          21688320
+  restored          21688320
+  delta             0
+
+tables
+  original          9
+  restored          9
+  delta             0
+```
+
+After restoring the backup, the `litestream:verify` rake task will delete the restored database file. If you need the restored database file, use the `litestream:restore` rake task instead.
+
 ### Introspection
 
 Litestream offers a handful of commands that allow you to introspect the state of your replication. The gem provides a few rake tasks that wrap these commands for you. For example, you can list the databases that Litestream is configured to replicate:

data/lib/litestream/commands.rb

@@ -17,6 +17,9 @@ module Litestream
     # raised when a litestream command requires a database argument but it isn't provided
     DatabaseRequiredException = Class.new(StandardError)
 
+    # raised when litestream fails to restore a database backup
+    BackupFailedException = Class.new(StandardError)
+
     class << self
       def platform
         [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
@@ -73,44 +76,67 @@ module Litestream
         exe_file
       end
 
-      def replicate(argv = {})
-        execute("replicate", argv)
+      def replicate(async: true, **argv)
+        execute("replicate", argv, async: async)
       end
 
-      def restore(database, argv = {})
+      def restore(database, async: true, **argv)
         raise DatabaseRequiredException, "database argument is required for restore command, e.g. litestream:restore -- --database=path/to/database.sqlite" if database.nil?
 
         dir, file = File.split(database)
         ext = File.extname(file)
         base = File.basename(file, ext)
         now = Time.now.utc.strftime("%Y%m%d%H%M%S")
+        backup = File.join(dir, "#{base}-#{now}#{ext}")
 
         args = {
-          "-o" => File.join(dir, "#{base}-#{now}#{ext}")
+          "-o" => backup
         }.merge(argv)
 
-        execute("restore", args, database)
+        execute("restore", args, database, async: async)
+
+        backup
       end
 
-      def databases(argv = {})
-        execute("databases", argv)
+      def databases(async: true, **argv)
+        execute("databases", argv, async: async)
       end
 
-      def generations(database, argv = {})
+      def generations(database, async: true, **argv)
         raise DatabaseRequiredException, "database argument is required for generations command, e.g. litestream:generations -- --database=path/to/database.sqlite" if database.nil?
 
-        execute("generations", argv, database)
+        execute("generations", argv, database, async: async)
       end
 
-      def snapshots(database, argv = {})
+      def snapshots(database, async: true, **argv)
         raise DatabaseRequiredException, "database argument is required for snapshots command, e.g. litestream:snapshots -- --database=path/to/database.sqlite" if database.nil?
 
-        execute("snapshots", argv, database)
+        execute("snapshots", argv, database, async: async)
+      end
+
+      def verify(database, async: true, **argv)
+        raise DatabaseRequiredException, "database argument is required for verify command, e.g. litestream:verify -- --database=path/to/database.sqlite" if database.nil? || !File.exist?(database)
+
+        backup = restore(database, async: false, **argv)
+
+        raise BackupFailedException, "Failed to create backup for validation" unless File.exist?(backup)
+
+        restored_tables_count = `sqlite3 #{backup} "select count(*) from sqlite_schema where type='table';"`.chomp.to_i
+        restored_size = File.size(backup)
+        original_tables_count = `sqlite3 #{database} "select count(*) from sqlite_schema where type='table';"`.chomp.to_i
+        original_size = File.size(database)
+
+        Dir.glob(backup + "*").each { |file| File.delete(file) }
+
+        {
+          size: {original: original_size, restored: restored_size},
+          tables: {original: original_tables_count, restored: restored_tables_count}
+        }
       end
 
       private
 
-      def execute(command, argv = {}, database = nil)
+      def execute(command, argv = {}, database = nil, async: true)
         if Litestream.configuration
           ENV["LITESTREAM_REPLICA_BUCKET"] ||= Litestream.configuration.replica_bucket
           ENV["LITESTREAM_ACCESS_KEY_ID"] ||= Litestream.configuration.replica_key_id
@@ -121,12 +147,14 @@ module Litestream
           "--config" => Rails.root.join("config", "litestream.yml").to_s
         }.merge(argv).to_a.flatten.compact
         cmd = [executable, command, *args, database].compact
+        puts cmd.inspect if ENV["DEBUG"]
 
-        # To release the resources of the Ruby process, just fork and exit.
-        # The forked process executes litestream and replaces itself.
-        if fork.nil?
-          puts cmd.inspect if ENV["DEBUG"]
-          exec(*cmd)
+        if async
+          # To release the resources of the Ruby process, just fork and exit.
+          # The forked process executes litestream and replaces itself.
+          exec(*cmd) if fork.nil?
+        else
+          system(*cmd)
         end
       end
     end

data/lib/litestream/version.rb

@@ -1,3 +1,3 @@
 module Litestream
-  VERSION = "0.5.1"
+  VERSION = "0.5.2"
 end

data/lib/tasks/litestream_tasks.rake

@@ -72,4 +72,29 @@ namespace :litestream do
 
     Litestream::Commands.snapshots(options.delete("--database") || options.delete("-database"), options)
   end
+
+  desc "verify backup of SQLite database from a Litestream replica, e.g. rake litestream:verify -- -database=storage/production.sqlite3"
+  task verify: :environment do
+    options = {}
+    if (separator_index = ARGV.index("--"))
+      ARGV.slice(separator_index + 1, ARGV.length)
+        .map { |pair| pair.split("=") }
+        .each { |opt| options[opt[0]] = opt[1] || nil }
+    end
+
+    result = Litestream::Commands.verify(options.delete("--database") || options.delete("-database"), options)
+
+    puts <<~TXT if result
+
+      size
+        original          #{result[:size][:original]}
+        restored          #{result[:size][:restored]}
+        delta             #{result[:size][:original] - result[:size][:restored]}
+
+      tables
+        original          #{result[:tables][:original]}
+        restored          #{result[:tables][:restored]}
+        delta             #{result[:tables][:original] - result[:tables][:restored]}
+    TXT
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: litestream
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Stephen Margheim