litestream 0.5.0 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8da245a396bc819cf0aa9bfae9a87f65a92047fd45a96bd8f8602eee6248adb6
-  data.tar.gz: 4bee567fab0ad7ab71d72c36091be4a32c4685ddfe1b69f6403c36ea927af437
+  metadata.gz: 4b38d63aa3c23ba58c57dd91f178e069e898519f06b38e7ece5d0f9b9238f20d
+  data.tar.gz: 730e40b5ba2fd39a726902052333c049943adebbde252d2a769013585172d789
 SHA512:
-  metadata.gz: 5b1ba5a3de6be15243513f10f37657c940cbd59471ae008434c73aa0a09b62869b7e940388003a254431ab503fafdc891f7cffcc95afcfa61c07e1f657617225
-  data.tar.gz: 34410a43ee4ab617e160a76989d4721d7c6b1861aa15d623602a7088d1b561266e40f2243e7c0a1d880778031d15b9ce71384ae1cf4a245494f21d2ea93426fc
+  metadata.gz: b07f6c0cd590a5ec095f778a1b473a6168adc5e747aadfbef2ce8dbea4047ef27ed9861ce0c36ac4d78bcdf7ea819a4e2d57db4c989ee8da9190523a77ae86cf
+  data.tar.gz: 0f9436ff88c9adf3b521c853e3a8e770b8aea4590fd2d8eb2cb575d48dfd7044a6156982b2e2924724c1bd5a5b97840eba567b143084abaf28786a4e50523135

data/README.md

@@ -165,6 +165,81 @@ You can forward arguments in whatever order you like, you simply need to ensure
     Disables environment variable expansion in configuration file.
 ```
 
+### Verification
+
+You can verify the integrity of your backed-up databases using the gem's provided `litestream:verify` rake task. This rake task requires that you specify which specific database you want to verify. As with the `litestream:restore` tasks, you pass arguments to the rake task via argument forwarding. For example, to verify the production database, you would run:
+
+```shell
+bin/rails litestream:verify -- --database=storage/production.sqlite3
+# or
+bundle exec rake litestream:verify -- --database=storage/production.sqlite3
+```
+
+The `litestream:verify` rake task takes the same options as the `litestream:restore` rake task. After restoring the backup, the rake task will verify the integrity of the restored database by ensuring that the restored database file
+
+1. exists,
+2. can be opened by SQLite, and
+3. sufficiently matches the original database file.
+
+Since point 3 is subjective, the rake task will output a message providing both the file size and number of tables of both the "original" and "restored" databases. You must manually verify that the restored database is within an acceptable range of the original database.
+
+The rake task will output a message similar to the following:
+
+```
+size
+  original          21688320
+  restored          21688320
+  delta             0
+
+tables
+  original          9
+  restored          9
+  delta             0
+```
+
+After restoring the backup, the `litestream:verify` rake task will delete the restored database file. If you need the restored database file, use the `litestream:restore` rake task instead.
+
+### Introspection
+
+Litestream offers a handful of commands that allow you to introspect the state of your replication. The gem provides a few rake tasks that wrap these commands for you. For example, you can list the databases that Litestream is configured to replicate:
+
+```shell
+bin/rails litestream:databases
+```
+
+This will return a list of databases and their configured replicas:
+
+```
+path                                                 replicas
+/Users/you/Code/your-app/storage/production.sqlite3  s3
+```
+
+You can also list the generations of a specific database:
+
+```shell
+bin/rails litestream:generations -- --database=storage/production.sqlite3
+```
+
+This will list all generations for the specified database, including stats about their lag behind the primary database and the time range they cover.
+
+```
+name  generation        lag     start                 end
+s3    a295b16a796689f3  -156ms  2024-04-17T00:01:19Z  2024-04-17T00:01:19Z
+```
+
+Finally, you can list the snapshots available for a database:
+
+```shell
+bin/rails litestream:snapshots -- --database=storage/production.sqlite3
+```
+
+This command lists snapshots available for that specified database:
+
+```
+replica  generation        index  size     created
+s3       a295b16a796689f3  1      4645465  2024-04-17T00:01:19Z
+```
+
 ### Additional commands
 
 The rake tasks are the recommended way to interact with the Litestream utility in your Rails application or Ruby project. But, you _can_ work directly with the Litestream CLI. Since the gem installs the native executable via Bundler, the `litestream` command will be available in your `PATH`.

data/lib/litestream/commands.rb

@@ -14,106 +14,148 @@ module Litestream
     # raised when LITESTREAM_INSTALL_DIR does not exist
     DirectoryNotFoundException = Class.new(StandardError)
 
-    def self.platform
-      [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
-    end
+    # raised when a litestream command requires a database argument but it isn't provided
+    DatabaseRequiredException = Class.new(StandardError)
+
+    # raised when litestream fails to restore a database backup
+    BackupFailedException = Class.new(StandardError)
+
+    class << self
+      def platform
+        [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
+      end
 
-    def self.executable(exe_path: DEFAULT_DIR)
-      litestream_install_dir = ENV["LITESTREAM_INSTALL_DIR"]
-      if litestream_install_dir
-        if File.directory?(litestream_install_dir)
-          warn "NOTE: using LITESTREAM_INSTALL_DIR to find litestream executable: #{litestream_install_dir}"
-          exe_path = litestream_install_dir
-          exe_file = File.expand_path(File.join(litestream_install_dir, "litestream"))
+      def executable(exe_path: DEFAULT_DIR)
+        litestream_install_dir = ENV["LITESTREAM_INSTALL_DIR"]
+        if litestream_install_dir
+          if File.directory?(litestream_install_dir)
+            warn "NOTE: using LITESTREAM_INSTALL_DIR to find litestream executable: #{litestream_install_dir}"
+            exe_path = litestream_install_dir
+            exe_file = File.expand_path(File.join(litestream_install_dir, "litestream"))
+          else
+            raise DirectoryNotFoundException, <<~MESSAGE
+              LITESTREAM_INSTALL_DIR is set to #{litestream_install_dir}, but that directory does not exist.
+            MESSAGE
+          end
         else
-          raise DirectoryNotFoundException, <<~MESSAGE
-            LITESTREAM_INSTALL_DIR is set to #{litestream_install_dir}, but that directory does not exist.
-          MESSAGE
+          if Litestream::Upstream::NATIVE_PLATFORMS.keys.none? { |p| Gem::Platform.match_gem?(Gem::Platform.new(p), GEM_NAME) }
+            raise UnsupportedPlatformException, <<~MESSAGE
+              litestream-ruby does not support the #{platform} platform
+              Please install litestream following instructions at https://litestream.io/install
+            MESSAGE
+          end
+
+          exe_file = Dir.glob(File.expand_path(File.join(exe_path, "*", "litestream"))).find do |f|
+            Gem::Platform.match_gem?(Gem::Platform.new(File.basename(File.dirname(f))), GEM_NAME)
+          end
         end
-      else
-        if Litestream::Upstream::NATIVE_PLATFORMS.keys.none? { |p| Gem::Platform.match_gem?(Gem::Platform.new(p), GEM_NAME) }
-          raise UnsupportedPlatformException, <<~MESSAGE
-            litestream-ruby does not support the #{platform} platform
-            Please install litestream following instructions at https://litestream.io/install
+
+        if exe_file.nil? || !File.exist?(exe_file)
+          raise ExecutableNotFoundException, <<~MESSAGE
+            Cannot find the litestream executable for #{platform} in #{exe_path}
+
+            If you're using bundler, please make sure you're on the latest bundler version:
+
+                gem install bundler
+                bundle update --bundler
+
+            Then make sure your lock file includes this platform by running:
+
+                bundle lock --add-platform #{platform}
+                bundle install
+
+            See `bundle lock --help` output for details.
+
+            If you're still seeing this message after taking those steps, try running
+            `bundle config` and ensure `force_ruby_platform` isn't set to `true`. See
+            https://github.com/fractaledmind/litestream-ruby#check-bundle_force_ruby_platform
+            for more details.
           MESSAGE
         end
 
-        exe_file = Dir.glob(File.expand_path(File.join(exe_path, "*", "litestream"))).find do |f|
-          Gem::Platform.match_gem?(Gem::Platform.new(File.basename(File.dirname(f))), GEM_NAME)
-        end
+        exe_file
       end
 
-      if exe_file.nil? || !File.exist?(exe_file)
-        raise ExecutableNotFoundException, <<~MESSAGE
-          Cannot find the litestream executable for #{platform} in #{exe_path}
+      def replicate(async: true, **argv)
+        execute("replicate", argv, async: async)
+      end
 
-          If you're using bundler, please make sure you're on the latest bundler version:
+      def restore(database, async: true, **argv)
+        raise DatabaseRequiredException, "database argument is required for restore command, e.g. litestream:restore -- --database=path/to/database.sqlite" if database.nil?
 
-              gem install bundler
-              bundle update --bundler
+        dir, file = File.split(database)
+        ext = File.extname(file)
+        base = File.basename(file, ext)
+        now = Time.now.utc.strftime("%Y%m%d%H%M%S")
+        backup = File.join(dir, "#{base}-#{now}#{ext}")
 
-          Then make sure your lock file includes this platform by running:
+        args = {
+          "-o" => backup
+        }.merge(argv)
 
-              bundle lock --add-platform #{platform}
-              bundle install
+        execute("restore", args, database, async: async)
 
-          See `bundle lock --help` output for details.
+        backup
+      end
 
-          If you're still seeing this message after taking those steps, try running
-          `bundle config` and ensure `force_ruby_platform` isn't set to `true`. See
-          https://github.com/fractaledmind/litestream-ruby#check-bundle_force_ruby_platform
-          for more details.
-        MESSAGE
+      def databases(async: true, **argv)
+        execute("databases", argv, async: async)
       end
 
-      exe_file
-    end
+      def generations(database, async: true, **argv)
+        raise DatabaseRequiredException, "database argument is required for generations command, e.g. litestream:generations -- --database=path/to/database.sqlite" if database.nil?
 
-    def self.replicate(argv = {})
-      if Litestream.configuration
-        ENV["LITESTREAM_REPLICA_BUCKET"] ||= Litestream.configuration.replica_bucket
-        ENV["LITESTREAM_ACCESS_KEY_ID"] ||= Litestream.configuration.replica_key_id
-        ENV["LITESTREAM_SECRET_ACCESS_KEY"] ||= Litestream.configuration.replica_access_key
+        execute("generations", argv, database, async: async)
       end
 
-      args = {
-        "--config" => Rails.root.join("config", "litestream.yml").to_s
-      }.merge(argv).to_a.flatten.compact
-
-      command = [executable, "replicate", *args]
-      puts command.inspect
+      def snapshots(database, async: true, **argv)
+        raise DatabaseRequiredException, "database argument is required for snapshots command, e.g. litestream:snapshots -- --database=path/to/database.sqlite" if database.nil?
 
-      # To release the resources of the Ruby process, just fork and exit.
-      # The forked process executes litestream and replaces itself.
-      if fork.nil?
-        exec(*command)
+        execute("snapshots", argv, database, async: async)
       end
-    end
 
-    def self.restore(database, argv = {})
-      if Litestream.configuration
-        ENV["LITESTREAM_REPLICA_BUCKET"] ||= Litestream.configuration.replica_bucket
-        ENV["LITESTREAM_ACCESS_KEY_ID"] ||= Litestream.configuration.replica_key_id
-        ENV["LITESTREAM_SECRET_ACCESS_KEY"] ||= Litestream.configuration.replica_access_key
+      def verify(database, async: true, **argv)
+        raise DatabaseRequiredException, "database argument is required for verify command, e.g. litestream:verify -- --database=path/to/database.sqlite" if database.nil? || !File.exist?(database)
+
+        backup = restore(database, async: false, **argv)
+
+        raise BackupFailedException, "Failed to create backup for validation" unless File.exist?(backup)
+
+        restored_tables_count = `sqlite3 #{backup} "select count(*) from sqlite_schema where type='table';"`.chomp.to_i
+        restored_size = File.size(backup)
+        original_tables_count = `sqlite3 #{database} "select count(*) from sqlite_schema where type='table';"`.chomp.to_i
+        original_size = File.size(database)
+
+        Dir.glob(backup + "*").each { |file| File.delete(file) }
+
+        {
+          size: {original: original_size, restored: restored_size},
+          tables: {original: original_tables_count, restored: restored_tables_count}
+        }
       end
 
-      dir, file = File.split(database)
-      ext = File.extname(file)
-      base = File.basename(file, ext)
-      now = Time.now.utc.strftime("%Y%m%d%H%M%S")
+      private
 
-      args = {
-        "--config" => Rails.root.join("config", "litestream.yml").to_s,
-        "-o" => File.join(dir, "#{base}-#{now}#{ext}")
-      }.merge(argv).to_a.flatten.compact
+      def execute(command, argv = {}, database = nil, async: true)
+        if Litestream.configuration
+          ENV["LITESTREAM_REPLICA_BUCKET"] ||= Litestream.configuration.replica_bucket
+          ENV["LITESTREAM_ACCESS_KEY_ID"] ||= Litestream.configuration.replica_key_id
+          ENV["LITESTREAM_SECRET_ACCESS_KEY"] ||= Litestream.configuration.replica_access_key
+        end
 
-      command = [executable, "restore", *args, database]
-      puts command.inspect
+        args = {
+          "--config" => Rails.root.join("config", "litestream.yml").to_s
+        }.merge(argv).to_a.flatten.compact
+        cmd = [executable, command, *args, database].compact
+        puts cmd.inspect if ENV["DEBUG"]
 
-      # To release the resources of the Ruby process, just fork and exit.
-      # The forked process executes litestream and replaces itself.
-      if fork.nil?
-        exec(*command)
+        if async
+          # To release the resources of the Ruby process, just fork and exit.
+          # The forked process executes litestream and replaces itself.
+          exec(*cmd) if fork.nil?
+        else
+          system(*cmd)
+        end
       end
     end
   end

data/lib/litestream/version.rb

@@ -1,3 +1,3 @@
 module Litestream
-  VERSION = "0.5.0"
+  VERSION = "0.5.2"
 end

data/lib/tasks/litestream_tasks.rake

@@ -36,4 +36,65 @@ namespace :litestream do
 
     Litestream::Commands.restore(options.delete("--database") || options.delete("-database"), options)
   end
+
+  desc "List all databases and associated replicas in the config file, e.g. rake litestream:databases -- -no-expand-env"
+  task databases: :environment do
+    options = {}
+    if (separator_index = ARGV.index("--"))
+      ARGV.slice(separator_index + 1, ARGV.length)
+        .map { |pair| pair.split("=") }
+        .each { |opt| options[opt[0]] = opt[1] || nil }
+    end
+
+    Litestream::Commands.databases(options)
+  end
+
+  desc "List all generations for a database or replica, e.g. rake litestream:generations -- -database=storage/production.sqlite3"
+  task generations: :environment do
+    options = {}
+    if (separator_index = ARGV.index("--"))
+      ARGV.slice(separator_index + 1, ARGV.length)
+        .map { |pair| pair.split("=") }
+        .each { |opt| options[opt[0]] = opt[1] || nil }
+    end
+
+    Litestream::Commands.generations(options.delete("--database") || options.delete("-database"), options)
+  end
+
+  desc "List all snapshots for a database or replica, e.g. rake litestream:snapshots -- -database=storage/production.sqlite3"
+  task snapshots: :environment do
+    options = {}
+    if (separator_index = ARGV.index("--"))
+      ARGV.slice(separator_index + 1, ARGV.length)
+        .map { |pair| pair.split("=") }
+        .each { |opt| options[opt[0]] = opt[1] || nil }
+    end
+
+    Litestream::Commands.snapshots(options.delete("--database") || options.delete("-database"), options)
+  end
+
+  desc "verify backup of SQLite database from a Litestream replica, e.g. rake litestream:verify -- -database=storage/production.sqlite3"
+  task verify: :environment do
+    options = {}
+    if (separator_index = ARGV.index("--"))
+      ARGV.slice(separator_index + 1, ARGV.length)
+        .map { |pair| pair.split("=") }
+        .each { |opt| options[opt[0]] = opt[1] || nil }
+    end
+
+    result = Litestream::Commands.verify(options.delete("--database") || options.delete("-database"), options)
+
+    puts <<~TXT if result
+
+      size
+        original          #{result[:size][:original]}
+        restored          #{result[:size][:restored]}
+        delta             #{result[:size][:original] - result[:size][:restored]}
+
+      tables
+        original          #{result[:tables][:original]}
+        restored          #{result[:tables][:restored]}
+        delta             #{result[:tables][:original] - result[:tables][:restored]}
+    TXT
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: litestream
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.2
 platform: ruby
 authors:
 - Stephen Margheim