litestream 0.9.0-x86_64-darwin → 0.10.0-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc937c14c8cdbc08db031431903637338b95fed9cee32fcffa945da97bcc5927
-  data.tar.gz: 8b305efd28e72ef7316fc53e790ca10ef29cc317d74fd72474ed4007a0469936
+  metadata.gz: 40caaac876c83fb253b5be358ee51822a27788fd789531f058c6945014716726
+  data.tar.gz: 49650381680cb1f359c5945e1b7d9b38dca95cfdd446fad0a0534a357ea7686d
 SHA512:
-  metadata.gz: 672231f8716e812ee516d13cfab97386001cd9084f2aa03418d1f17838bbaaa8442e6ccc4ddf3c3edd654743e2f2f11b8f72b5d887f79060d405ac03d2de9832
-  data.tar.gz: e19b81794bfc2fae07aff2bc1b6c9084a10bd309fda38f2a1f8340791262475fd4d0a5171bd067322a4342c31bb69fcfbf8eea9fd27ec497b76b37dd3bf746ad
+  metadata.gz: 53bddbe69e324e5bf376192578b4a34d9e0a7780e8cb08692e06e0419bd7bded4058597f51a1fc8e8d4909d3f57bf11e3fb1a2d25c025823a21520b0522b2196
+  data.tar.gz: a0a1392997dd3f2aa69a6a7805b7838f1b87e84b7e23bdc1ccf5c84a689b5da66e327ff7a0126502ec7a3db9fb9746f5c31db8b0a3796472f5ae916039f67b32

data/README.md

@@ -6,17 +6,23 @@
 
 Install the gem and add to the application's Gemfile by executing:
 
-    $ bundle add litestream
+```sh
+bundle add litestream
+```
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
-    $ gem install litestream
+```sh
+gem install litestream
+```
 
 After installing the gem, run the installer:
 
-    $ rails generate litestream:install
+```sh
+rails generate litestream:install
+```
 
-The installer will create a configuration file at `config/litestream.yml`, an initializer file for configuring the gem at `config/initializers/litestream.rb`, as well as a `Procfile` so that you can run the Litestream replication process alongside your Rails application in production.
+The installer will create a configuration file at `config/litestream.yml` and an initializer file for configuring the gem at `config/initializers/litestream.rb`.
 
 This gem wraps the standalone executable version of the [Litestream](https://litestream.io/install/source/) utility. These executables are platform specific, so there are actually separate underlying gems per platform, but the correct gem will automatically be picked for your platform. Litestream itself doesn't support Windows, so this gem doesn't either.
 
@@ -77,14 +83,28 @@ However, if you need manual control over the Litestream configuration, you can m
 
 ### Replication
 
-By default, the gem will create or append to a `Procfile` to start the Litestream process via the gem's provided `litestream:replicate` rake task. This rake task will automatically load the configuration file and set the environment variables before starting the Litestream process. You can also execute this rake task yourself:
+In order to stream changes to your configured replicas, you need to start the Litestream replication process.
 
-```shell
-bin/rails litestream:replicate
-# or
-bundle exec rake litestream:replicate
+The simplest way to run the Litestream replication process is use the Puma plugin provided by the gem. This allows you to run the Litestream replication process together with Puma and have Puma monitor and manage it. You just need to add
+
+```ruby
+plugin :litestream
 ```
 
+to your `puma.rb` configuration.
+
+If you would prefer to run the Litestream replication process separately from Puma, you can use the provided `litestream:replicate` rake task. This rake task will automatically load the configuration file and set the environment variables before starting the Litestream process.
+
+The simplest way to spin up a Litestream process separately from your Rails application is to use a `Procfile`:
+
+```yaml
+# Procfile
+rails: bundle exec rails server --port $PORT
+litestream: bin/rails litestream:replicate
+```
+
+Alternatively, you could setup a `systemd` service to manage the Litestream replication process, but setting this up is outside the scope of this README.
+
 If you need to pass arguments through the rake task to the underlying `litestream` command, that can be done with argument forwarding:
 
 ```shell
@@ -94,6 +114,7 @@ bin/rails litestream:replicate -- -exec "foreman start"
 This example utilizes the `-exec` option available on [the `replicate` command](https://litestream.io/reference/replicate/) which provides basic process management, since Litestream will exit when the child process exits. In this example, we only launch our collection of Rails application processes (like Rails and SolidQueue, for example) after the Litestream replication process is ready.
 
 The Litestream `replicate` command supports the following options, which can be passed through the rake task:
+
 ```shell
 -config PATH
     Specifies the configuration file.
@@ -168,37 +189,19 @@ You can forward arguments in whatever order you like, you simply need to ensure
 
 ### Verification
 
-You can verify the integrity of your backed-up databases using the gem's provided `litestream:verify` rake task. This rake task requires that you specify which specific database you want to verify. As with the `litestream:restore` tasks, you pass arguments to the rake task via argument forwarding. For example, to verify the production database, you would run:
+You can verify the integrity of your backed-up databases using the gem's provided `Litestream.verify!` method. The method takes the path to a database file that you have configured Litestream to backup; that is, it takes one of the `path` values under the `dbs` key in your `litestream.yml` configuration file. For example, to verify the production database, you would run:
 
-```shell
-bin/rails litestream:verify -- --database=storage/production.sqlite3
-# or
-bundle exec rake litestream:verify -- --database=storage/production.sqlite3
+```ruby
+Litestream.verify! "storage/production.sqlite3"
 ```
 
-The `litestream:verify` rake task takes the same options as the `litestream:restore` rake task. After restoring the backup, the rake task will verify the integrity of the restored database by ensuring that the restored database file
+In order to verify that the backup for that database is both restorable and fresh, the method will add a new row to that database under the `_litestream_verification` table, which it will create if needed. It will then wait 10 seconds to give the Litestream utility time to replicate that change to whatever storage providers you have configured. After that, it will download the latest backup from that storage provider and ensure that this verification row is present in the backup. If the verification row is _not_ present, the method will raise a `Litestream::VerificationFailure` exception. This check ensures that the restored database file
 
 1. exists,
 2. can be opened by SQLite, and
-3. sufficiently matches the original database file.
-
-Since point 3 is subjective, the rake task will output a message providing both the file size and number of tables of both the "original" and "restored" databases. You must manually verify that the restored database is within an acceptable range of the original database.
-
-The rake task will output a message similar to the following:
-
-```
-size
-  original          21688320
-  restored          21688320
-  delta             0
-
-tables
-  original          9
-  restored          9
-  delta             0
-```
+3. has up-to-date data.
 
-After restoring the backup, the `litestream:verify` rake task will delete the restored database file. If you need the restored database file, use the `litestream:restore` rake task instead.
+After restoring the backup, the `Litestream.verify!` method will delete the restored database file. If you need the restored database file, use the `litestream:restore` rake task or `Litestream::Commands.restore` method instead.
 
 ### Introspection
 

data/lib/litestream/commands.rb

@@ -88,46 +88,6 @@ module Litestream
         execute("restore", argv, database, async: async, tabled_output: false)
       end
 
-      def verify(database, async: false, **argv)
-        raise DatabaseRequiredException, "database argument is required for verify command, e.g. litestream:verify -- --database=path/to/database.sqlite" if database.nil? || !File.exist?(database)
-        argv.stringify_keys!
-
-        dir, file = File.split(database)
-        ext = File.extname(file)
-        base = File.basename(file, ext)
-        now = Time.now.utc.strftime("%Y%m%d%H%M%S")
-        backup = File.join(dir, "#{base}-#{now}#{ext}")
-        args = {
-          "-o" => backup
-        }.merge(argv)
-        restore(database, async: false, **args)
-
-        restored_schema = `sqlite3 #{backup} "select name, type from sqlite_schema;"`.chomp.split("\n")
-        restored_data = restored_schema.map { _1.split("|") }.group_by(&:last)
-        restored_rows_count = restored_data["table"]&.sum { |tbl, _| `sqlite3 #{backup} "select count(*) from #{tbl};"`.chomp.to_i }
-
-        original_schema = `sqlite3 #{database} "select name, type from sqlite_schema;"`.chomp.split("\n")
-        original_data = original_schema.map { _1.split("|") }.group_by(&:last)
-        original_rows_count = original_data["table"]&.sum { |tbl, _| `sqlite3 #{database} "select count(*) from #{tbl};"`.chomp.to_i }
-
-        Dir.glob(backup + "*").each { |file| File.delete(file) }
-
-        {
-          "original" => {
-            "path" => database,
-            "tables" => original_data["table"]&.size,
-            "indexes" => original_data["index"]&.size,
-            "rows" => original_rows_count
-          },
-          "restored" => {
-            "path" => backup,
-            "tables" => restored_data["table"]&.size,
-            "indexes" => restored_data["index"]&.size,
-            "rows" => restored_rows_count
-          }
-        }
-      end
-
       def databases(async: false, **argv)
         execute("databases", argv, async: async, tabled_output: true)
       end

data/lib/litestream/{railtie.rb → engine.rb}

@@ -1,9 +1,13 @@
 # frozen_string_literal: true
 
-require "rails/railtie"
+require "rails/engine"
 
 module Litestream
-  class Railtie < ::Rails::Railtie
+  class Engine < ::Rails::Engine
+    isolate_namespace Litestream
+
+    config.litestream = ActiveSupport::OrderedOptions.new
+
     # Load the `litestream:install` generator into the host Rails app
     generators do
       require_relative "generators/litestream/install_generator"
@@ -13,5 +17,11 @@ module Litestream
     rake_tasks do
       load "tasks/litestream_tasks.rake"
     end
+
+    initializer "litestream.config" do
+      config.litestream.each do |name, value|
+        Litestream.public_send(:"#{name}=", value)
+      end
+    end
   end
 end

data/lib/litestream/generators/litestream/install_generator.rb

@@ -15,19 +15,6 @@ module Litestream
         template "initializer.rb", "config/initializers/litestream.rb"
       end
 
-      def create_or_update_procfile
-        if File.exist?("Procfile")
-          append_to_file "Procfile", "litestream: bin/rails litestream:replicate"
-        else
-          create_file "Procfile" do
-            <<~PROCFILE
-              rails: bundle exec rails server --port $PORT
-              litestream: bin/rails litestream:replicate
-            PROCFILE
-          end
-        end
-      end
-
       private
 
       def production_sqlite_databases

data/lib/litestream/version.rb

@@ -1,3 +1,3 @@
 module Litestream
-  VERSION = "0.9.0"
+  VERSION = "0.10.0"
 end

data/lib/litestream.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "sqlite3"
+
 module Litestream
   class << self
     attr_accessor :configuration
@@ -16,9 +18,118 @@ module Litestream
     def initialize
     end
   end
+
+  VerificationFailure = Class.new(StandardError)
+
+  mattr_writer :username
+  mattr_writer :password
+
+  class << self
+    def verify!(database_path)
+      database = SQLite3::Database.new(database_path)
+      database.execute("CREATE TABLE IF NOT EXISTS _litestream_verification (id INTEGER PRIMARY KEY, uuid BLOB)")
+      sentinel = SecureRandom.uuid
+      database.execute("INSERT INTO _litestream_verification (uuid) VALUES (?)", [sentinel])
+      # give the Litestream replication process time to replicate the sentinel value
+      sleep 10
+
+      backup_path = "tmp/#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_#{sentinel}.sqlite3"
+      Litestream::Commands.restore(database_path, **{"-o" => backup_path})
+
+      backup = SQLite3::Database.new(backup_path)
+      result = backup.execute("SELECT 1 FROM _litestream_verification WHERE uuid = ? LIMIT 1", sentinel) # => [[1]] || []
+
+      raise VerificationFailure, "Verification failed, sentinel not found" if result.empty?
+    ensure
+      database.execute("DELETE FROM _litestream_verification WHERE uuid = ?", sentinel)
+      database.close
+      Dir.glob(backup_path + "*").each { |file| File.delete(file) }
+    end
+
+    # use method instead of attr_accessor to ensure
+    # this works if variable set after Litestream is loaded
+    def username
+      @username ||= ENV["LITESTREAM_USERNAME"] || @@username
+    end
+
+    # use method instead of attr_accessor to ensure
+    # this works if variable set after Litestream is loaded
+    def password
+      @password ||= ENV["LITESTREAM_PASSWORD"] || @@password
+    end
+
+    def replicate_process
+      info = {}
+      if !`which systemctl`.empty?
+        systemctl_status = `systemctl status litestream`.chomp
+        # ["● litestream.service - Litestream",
+        #  "     Loaded: loaded (/lib/systemd/system/litestream.service; enabled; vendor preset: enabled)",
+        #  "     Active: active (running) since Tue 2023-07-25 13:49:43 UTC; 8 months 24 days ago",
+        #  "   Main PID: 1179656 (litestream)",
+        #  "      Tasks: 9 (limit: 1115)",
+        #  "     Memory: 22.9M",
+        #  "        CPU: 10h 49.843s",
+        #  "     CGroup: /system.slice/litestream.service",
+        #  "             └─1179656 /usr/bin/litestream replicate",
+        #  "",
+        #  "Warning: some journal files were not opened due to insufficient permissions."]
+        systemctl_status.split("\n").each do |line|
+          line.strip!
+          if line.start_with?("Main PID:")
+            _key, value = line.split(":")
+            pid, _name = value.strip.split(" ")
+            info[:pid] = pid
+          elsif line.start_with?("Active:")
+            _key, value = line.split(":")
+            value, _ago = value.split(";")
+            status, timestamp = value.split(" since ")
+            info[:started] = DateTime.strptime(timestamp.strip, "%Y-%m-%d %H:%M:%S %Z")
+            info[:status] = status.split("(").first.strip
+          end
+        end
+      else
+        litestream_replicate_ps = `ps -a | grep litestream | grep replicate`.chomp
+        litestream_replicate_ps.split("\n").each do |line|
+          next unless line.include?("litestream replicate")
+          pid, * = line.split(" ")
+          info[:pid] = pid
+          state, _, lstart = `ps -o "state,lstart" #{pid}`.chomp.split("\n").last.partition(/\s+/)
+
+          info[:status] = case state[0]
+          when "I" then "idle"
+          when "R" then "running"
+          when "S" then "sleeping"
+          when "T" then "stopped"
+          when "U" then "uninterruptible"
+          when "Z" then "zombie"
+          end
+          info[:started] = DateTime.strptime(lstart.strip, "%a %b %d %H:%M:%S %Y")
+        end
+      end
+      info
+    end
+
+    def databases
+      databases = Commands.databases
+
+      databases.each do |db|
+        generations = Commands.generations(db["path"])
+        snapshots = Commands.snapshots(db["path"])
+        db["path"] = db["path"].gsub(Rails.root.to_s, "[ROOT]")
+
+        db["generations"] = generations.map do |generation|
+          id = generation["generation"]
+          replica = generation["name"]
+          generation["snapshots"] = snapshots.select { |snapshot| snapshot["generation"] == id && snapshot["replica"] == replica }
+            .map { |s| s.slice("index", "size", "created") }
+          generation.slice("generation", "name", "lag", "start", "end", "snapshots")
+        end
+      end
+    end
+  end
 end
 
 require_relative "litestream/version"
 require_relative "litestream/upstream"
 require_relative "litestream/commands"
-require_relative "litestream/railtie" if defined?(::Rails::Railtie)
+require_relative "litestream/engine" if defined?(::Rails::Engine)

data/lib/puma/plugin/litestream.rb

@@ -0,0 +1,69 @@
+require "puma/plugin"
+
+# Copied from https://github.com/rails/solid_queue/blob/15408647f1780033dad223d3198761ea2e1e983e/lib/puma/plugin/solid_queue.rb
+Puma::Plugin.create do
+  attr_reader :puma_pid, :litestream_pid, :log_writer
+
+  def start(launcher)
+    @log_writer = launcher.log_writer
+    @puma_pid = $$
+
+    launcher.events.on_booted do
+      @litestream_pid = fork do
+        Thread.new { monitor_puma }
+        Litestream::Commands.replicate(async: true)
+      end
+
+      in_background do
+        monitor_litestream
+      end
+    end
+
+    launcher.events.on_stopped { stop_litestream }
+    launcher.events.on_restart { stop_litestream }
+  end
+
+  private
+
+  def stop_litestream
+    Process.waitpid(litestream_pid, Process::WNOHANG)
+    log_writer.log "Stopping Litestream..."
+    Process.kill(:INT, litestream_pid) if litestream_pid
+    Process.wait(litestream_pid)
+  rescue Errno::ECHILD, Errno::ESRCH
+  end
+
+  def monitor_puma
+    monitor(:puma_dead?, "Detected Puma has gone away, stopping Litestream...")
+  end
+
+  def monitor_litestream
+    monitor(:litestream_dead?, "Detected Litestream has gone away, stopping Puma...")
+  end
+
+  def monitor(process_dead, message)
+    loop do
+      if send(process_dead)
+        log message
+        Process.kill(:INT, $$)
+        break
+      end
+      sleep 2
+    end
+  end
+
+  def litestream_dead?
+    Process.waitpid(litestream_pid, Process::WNOHANG)
+    false
+  rescue Errno::ECHILD, Errno::ESRCH
+    true
+  end
+
+  def puma_dead?
+    Process.ppid != puma_pid
+  end
+
+  def log(...)
+    log_writer.log(...)
+  end
+end

data/lib/tasks/litestream_tasks.rake

@@ -80,66 +80,4 @@ namespace :litestream do
 
     Litestream::Commands.snapshots(database, async: true, **options)
   end
-
-  desc "verify backup of SQLite database from a Litestream replica, e.g. rake litestream:verify -- -database=storage/production.sqlite3"
-  task verify: :environment do
-    options = {}
-    if (separator_index = ARGV.index("--"))
-      ARGV.slice(separator_index + 1, ARGV.length)
-        .map { |pair| pair.split("=") }
-        .each { |opt| options[opt[0]] = opt[1] || nil }
-    end
-    database = options.delete("--database") || options.delete("-database")
-    options.symbolize_keys!
-
-    result = Litestream::Commands.verify(database, async: true, **options)
-    original_tables = result["original"]["tables"]
-    restored_tables = result["restored"]["tables"]
-    original_indexes = result["original"]["indexes"]
-    restored_indexes = result["restored"]["indexes"]
-    original_rows = result["original"]["rows"]
-    restored_rows = result["restored"]["rows"]
-
-    same_number_of_tables = original_tables == restored_tables
-    same_number_of_indexes = original_indexes == restored_indexes
-    same_number_of_rows = original_rows == restored_rows
-
-    if same_number_of_tables && same_number_of_indexes && same_number_of_rows
-      puts "Backup for `#{database}` verified as consistent!\n" + [
-        "  tables   #{original_tables}",
-        "  indexes  #{original_indexes}",
-        "  rows     #{original_rows}"
-      ].compact.join("\n")
-    else
-      abort "Verification failed for #{database}:\n" + [
-        (unless same_number_of_tables
-           if original_tables > restored_tables
-             diff = original_tables - restored_tables
-             "  Backup is missing #{diff} table#{"s" if diff > 1}"
-           else
-             diff = restored_tables - original_tables
-             "  Backup has extra #{diff} table#{"s" if diff > 1}"
-           end
-         end),
-        (unless same_number_of_indexes
-           if original_indexes > restored_indexes
-             diff = original_indexes - restored_indexes
-             "  Backup is missing #{diff} index#{"es" if diff > 1}"
-           else
-             diff = restored_indexes - original_indexes
-             "  Backup has extra #{diff} index#{"es" if diff > 1}"
-           end
-         end),
-        (unless same_number_of_rows
-           if original_rows > restored_rows
-             diff = original_rows - restored_rows
-             "  Backup is missing #{diff} row#{"s" if diff > 1}"
-           else
-             diff = restored_rows - original_rows
-             "  Backup has extra #{diff} row#{"s" if diff > 1}"
-           end
-         end)
-      ].compact.join("\n")
-    end
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: litestream
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: x86_64-darwin
 authors:
 - Stephen Margheim
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-04 00:00:00.000000000 Z
+date: 2024-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logfmt
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.10
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -82,12 +96,13 @@ files:
 - exe/x86_64-darwin/litestream
 - lib/litestream.rb
 - lib/litestream/commands.rb
+- lib/litestream/engine.rb
 - lib/litestream/generators/litestream/install_generator.rb
 - lib/litestream/generators/litestream/templates/config.yml.erb
 - lib/litestream/generators/litestream/templates/initializer.rb
-- lib/litestream/railtie.rb
 - lib/litestream/upstream.rb
 - lib/litestream/version.rb
+- lib/puma/plugin/litestream.rb
 - lib/tasks/litestream_tasks.rake
 homepage: https://github.com/fractaledmind/litestream-ruby
 licenses: