litestream 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e4c2bacf97ec888fa90d5182662278f0f7bef274e46334f087c48bc0cc0eb29f
-  data.tar.gz: 75ae8f8ba2566890c5c558d60a4ea4863ad59c449855685422e0edad995acf09
+  metadata.gz: 006c5aaaf41fb59b8af8178c090aab72e78e8c586c83c59ddc25f25bc232ea98
+  data.tar.gz: a320d9128e3f5ad4c757c3463edebdbaecc7542c48008e5b1efa75dc53bdab1a
 SHA512:
-  metadata.gz: de32e013065e37e3705306c69d380031f0dc9f33840a11ebc7846891bccd7b68c82fe02fea90c45648fdfd67a5b1709ef26fab35c035e2a679f708002ef447df
-  data.tar.gz: 730f7052e41485a5d8a81b324a6093fcbf7dadd5184571f4c81d60ca43f52344f7ed3a492f317c57f6205fc2b3f67f352afd123d17444f39b5ac1bfc4dd4a994
+  metadata.gz: f75e2d4e857f186de61a3665775d2018373509a4beeb448b11f404a5fa9b55d92da06eda2a75d09304d696b32f3e6e1d73423b851e44a77e7d31fe5bb4813cdd
+  data.tar.gz: abc842f2de56524cb9370eed3433c49491dc10adfe23f544b25ecd9017c8008ecbe378efe30a216340f97e6b79c9a138f8e88c03f4695c724427c6d12b22e259

data/README.md

@@ -434,6 +434,8 @@ s3       a295b16a796689f3  1      0         2036     2024-04-17T00:01:19Z
 
 In addition to the provided rake tasks, you can also run Litestream commands directly from Ruby. The gem provides a `Litestream::Commands` module that wraps the Litestream CLI commands. This is particularly useful for the introspection commands, as you can use the output in your Ruby code.
 
+Commands raise `Litestream::Commands::CommandFailedException` when the Litestream binary exits non-zero, and the exception message includes stderr. Pass `json: true` to receive parsed JSON when using Litestream 0.5 or newer (Litestream 0.3 does not support the `-json` flag), and use `timeout:` to limit how many seconds a command may run.
+
 The `Litestream::Commands.databases` method returns an array of hashes with the "path" and "replicas" keys for each database:
 
 ```ruby

data/app/controllers/litestream/application_controller.rb

@@ -1,5 +1,7 @@
 module Litestream
   class ApplicationController < Litestream.base_controller_class.constantize
+    layout "litestream/application"
+
     protect_from_forgery with: :exception
 
     if Litestream.password

data/lib/litestream/commands.rb

@@ -1,3 +1,5 @@
+require "json"
+require "open3"
 require_relative "upstream"
 
 module Litestream
@@ -20,6 +22,9 @@ module Litestream
     # raised when a litestream command fails
     CommandFailedException = Class.new(StandardError)
 
+    # raised when a litestream command times out
+    CommandTimeoutException = Class.new(CommandFailedException)
+
     module Output
       class << self
         def format(data)
@@ -47,7 +52,10 @@ module Litestream
         litestream_install_dir = ENV["LITESTREAM_INSTALL_DIR"]
         if litestream_install_dir
           if File.directory?(litestream_install_dir)
-            warn "NOTE: using LITESTREAM_INSTALL_DIR to find litestream executable: #{litestream_install_dir}"
+            unless @litestream_install_dir_noted
+              warn "NOTE: using LITESTREAM_INSTALL_DIR to find litestream executable: #{litestream_install_dir}"
+              @litestream_install_dir_noted = true
+            end
             exe_path = litestream_install_dir
             exe_file = File.expand_path(File.join(litestream_install_dir, "litestream"))
           else
@@ -94,12 +102,14 @@ module Litestream
         exe_file
       end
 
-      # Replicate can be run either as a fork or in the same process, depending on the context.
-      # Puma will start replication as a forked process, while running replication from a rake
-      # tasks won't.
+      # Replicate runs either as a child process or in the foreground, depending on the context.
+      # The Puma plugin starts it as a child and gets its pid back; the rake task runs it in the
+      # foreground and streams its output.
       def replicate(async: false, **argv)
         cmd = prepare("replicate", argv)
         run_replicate(cmd, async: async)
+      rescue CommandFailedException
+        raise
       rescue
         raise CommandFailedException, "Failed to execute `#{cmd.join(" ")}`"
       end
@@ -135,14 +145,19 @@ module Litestream
       private
 
       def execute(command, argv = {}, database = nil, tabled_output: true)
-        cmd = prepare(command, argv, database)
-        results = run(cmd, tabled_output: tabled_output)
-
-        if Array === results && results.one? && results[0]["level"] == "ERROR"
-          raise CommandFailedException, "Failed to execute `#{cmd.join(" ")}`; Reason: #{results[0]["error"]}"
+        argv = argv.stringify_keys
+        timeout = argv.delete("timeout")
+        output = if argv.delete("json")
+          argv["-json"] = nil
+          :json
+        elsif tabled_output
+          :table
         else
-          results
+          :raw
         end
+
+        cmd = prepare(command, argv, database)
+        run(cmd, output: output, timeout: timeout)
       end
 
       def prepare(command, argv = {}, database = nil)
@@ -154,29 +169,99 @@ module Litestream
 
         args = {
           "--config" => Litestream.config_path.to_s
-        }.merge(argv.stringify_keys).to_a.flatten.compact
-        cmd = [executable, command, *args, database].compact
+        }.merge(argv.stringify_keys).to_a.flatten.compact.map(&:to_s)
+        cmd = [executable, command, *args, database].compact.map(&:to_s)
         puts cmd.inspect if ENV["DEBUG"]
 
         cmd
       end
 
-      def run(cmd, tabled_output:)
-        stdout = `#{cmd.join(" ")}`.chomp
-        return stdout unless tabled_output
+      # Runs the command without a shell and returns its parsed stdout. A non-zero
+      # exit raises with stderr. With a timeout, the command runs in its own
+      # process group and is killed (TERM, then KILL) when the deadline passes.
+      def run(cmd, output:, timeout: nil)
+        stdin, stdout, stderr, wait_thread = Open3.popen3(*cmd, pgroup: true)
+        stdin.close
+        stdout_reader = Thread.new { stdout.read }
+        stderr_reader = Thread.new { stderr.read }
+
+        # The readers finish when the last process holding the pipes exits, so
+        # waiting on them covers descendants the direct child may have left behind.
+        unless wait_thread.join(timeout) && stdout_reader.join(timeout) && stderr_reader.join(timeout)
+          kill_process_group("TERM", wait_thread.pid)
+          wait_thread.join(1)
+          kill_process_group("KILL", wait_thread.pid)
+          wait_thread.join
+          [stdout_reader, stderr_reader].each(&:join)
+          raise CommandTimeoutException, "Failed to execute `#{cmd[1]}`: timed out after #{timeout} seconds"
+        end
+
+        status = wait_thread.value
+        unless status.success?
+          raise CommandFailedException, "Failed to execute `#{cmd[1]}` (exit status #{status.exitstatus}): #{stderr_reader.value.strip[0, 500]}"
+        end
+
+        case output
+        when :json then parse_json(cmd, stdout_reader.value)
+        when :table then parse_table(stdout_reader.value)
+        else stdout_reader.value
+        end
+      ensure
+        [stdout_reader, stderr_reader].each { |reader| reader&.join }
+        [stdin, stdout, stderr].each { |io| io&.close unless io&.closed? }
+      end
+
+      # macOS answers EPERM rather than ESRCH when every process left in the
+      # group has exited but not yet been reaped.
+      def kill_process_group(signal, pid)
+        Process.kill(signal, -pid)
+      rescue Errno::ESRCH, Errno::EPERM
+      end
+
+      # Two opt-in restore skips (-if-db-not-exists when the output exists,
+      # -if-replica-exists with no backups) exit 0 and print one logfmt line on
+      # stdout instead of JSON. They come back as {"skipped" => true, "message" => ...}.
+      def parse_json(cmd, stdout)
+        stdout = stdout.strip
+        return if stdout.empty?
+        return JSON.parse(stdout) if stdout.start_with?("{", "[")
+
+        skipped = stdout.match(/\Atime=\S+ level=\S+ msg=(?:"([^"]*)"|(\S+))/)
+        return {"skipped" => true, "message" => skipped[1] || skipped[2]} if skipped
+
+        raise CommandFailedException, "Unexpected output from `#{cmd[1]}`: #{stdout.lines.first.to_s.strip[0, 200]}"
+      end
+
+      def parse_table(stdout)
+        keys, *rows = stdout.strip.split("\n").map { _1.split(/\s+/) }
+        return [] unless keys
 
-        keys, *rows = stdout.split("\n").map { _1.split(/\s+/) }
         rows.map { keys.zip(_1).to_h }
       end
 
+      # Async replication returns the pid of the litestream process itself so a
+      # supervisor can signal and wait on it. Process.spawn is fork-and-exec in
+      # one call, so the pid it returns is the one that becomes litestream. The
+      # previous `exec(*cmd) if fork.nil?` also execs in a child but returns
+      # into the parent too, and the Puma plugin wrapped that in a fork of its
+      # own, so it recorded a pid that exited immediately while the daemon was
+      # reparented to init and outlived Puma.
       def run_replicate(cmd, async:)
         if async
-          exec(*cmd) if fork.nil?
+          Process.spawn(*cmd)
         else
           # When running in-process, we capture output continuously and write to stdout.
           IO.popen(cmd, err: [:child, :out]) do |io|
             io.each_line { |line| puts line }
           end
+          status = $?
+          # `replicate` runs until it is signalled, so a signal is how it is
+          # meant to end. A non-zero exit is litestream failing to start or
+          # dying on its own, which is invisible today: the task exits 0 and a
+          # supervisor sees a clean stop rather than a crash to restart.
+          if status && !status.success? && !status.signaled?
+            raise CommandFailedException, "Failed to execute `replicate` (exit status #{status.exitstatus})"
+          end
         end
       end
     end

data/lib/litestream/version.rb

@@ -1,3 +1,3 @@
 module Litestream
-  VERSION = "0.14.0"
+  VERSION = "0.15.0"
 end

data/lib/litestream.rb

@@ -1,10 +1,23 @@
 # frozen_string_literal: true
 
+require "etc"
 require "sqlite3"
+require "time"
 
 module Litestream
   VerificationFailure = Class.new(StandardError)
 
+  PROCESS_STATES = {
+    "D" => "uninterruptible",
+    "I" => "idle",
+    "R" => "running",
+    "S" => "sleeping",
+    "T" => "stopped",
+    "U" => "uninterruptible",
+    "Z" => "zombie"
+  }.freeze
+  private_constant :PROCESS_STATES
+
   class << self
     attr_writer :configuration
 
@@ -163,30 +176,58 @@ module Litestream
       info
     end
 
+    # Linux has every process under /proc, even in images that ship no `ps`
+    # (Debian slim) or BusyBox's, which lacks `-x` and `lstart` (Alpine).
     def process_info
-      litestream_replicate_ps = `ps -ax | grep litestream | grep replicate`
-      exit_code = $?.exitstatus
-      return unless exit_code.zero?
+      File.directory?("/proc/self") ? proc_process_info : ps_process_info
+    end
 
-      info = {}
-      litestream_replicate_ps.chomp.split("\n").each do |line|
-        next unless line.include?("litestream replicate")
-
-        pid, * = line.split(" ")
-        info[:pid] = pid
-        state, _, lstart = `ps -o "state,lstart" #{pid}`.chomp.split("\n").last.partition(/\s+/)
-
-        info[:status] = case state[0]
-        when "I" then "idle"
-        when "R" then "running"
-        when "S" then "sleeping"
-        when "T" then "stopped"
-        when "U" then "uninterruptible"
-        when "Z" then "zombie"
-        end
-        info[:started] = DateTime.strptime(lstart.strip, "%a %b %d %H:%M:%S %Y")
+    def proc_process_info(proc_dir = "/proc")
+      boot_time = File.foreach(File.join(proc_dir, "stat")).grep(/\Abtime /).first.split.last.to_i
+
+      Dir.glob(File.join(proc_dir, "[0-9]*")).each do |dir|
+        executable, command = File.read(File.join(dir, "cmdline")).split("\0")
+        next unless File.basename(executable.to_s) == "litestream" && command == "replicate"
+
+        # Fields are counted after the command name's closing paren, since the
+        # name itself may contain spaces and parens.
+        state, *fields = File.read(File.join(dir, "stat")).rpartition(")").last.split
+        start_ticks = fields[18].to_i
+
+        return {
+          pid: File.basename(dir),
+          status: PROCESS_STATES[state[0]],
+          started: Time.at(boot_time + start_ticks.fdiv(Etc.sysconf(Etc::SC_CLK_TCK))).to_datetime
+        }
+      rescue SystemCallError
+        # the process exited while being read, or belongs to another user
+        next
       end
-      info
+      nil
+    end
+
+    # macOS. The command name is matched on its own so that a shell or editor
+    # whose arguments mention "litestream replicate" is not mistaken for it.
+    def ps_process_info
+      output = `ps -axo pid=,stat=,lstart=,comm=`
+      return unless $?.success?
+
+      output.each_line do |line|
+        # lstart is always five words ("Mon Jul  1 11:10:58 2024"); comm is last
+        # because it may contain spaces.
+        pid, state, *lstart, command = line.strip.split(" ", 8)
+        next unless File.basename(command.to_s) == "litestream"
+        next unless `ps -o args= -p #{pid}`.match?(/litestream replicate(?: |$)/)
+
+        return {
+          pid: pid,
+          status: PROCESS_STATES[state[0]],
+          started: Time.strptime(lstart.join(" "), "%a %b %d %H:%M:%S %Y").to_datetime
+        }
+      end
+      nil
+    rescue Errno::ENOENT
+      nil
     end
   end
 end

data/lib/puma/plugin/litestream.rb

@@ -2,17 +2,13 @@ require "puma/plugin"
 
 # Copied from https://github.com/rails/solid_queue/blob/15408647f1780033dad223d3198761ea2e1e983e/lib/puma/plugin/solid_queue.rb
 Puma::Plugin.create do
-  attr_reader :puma_pid, :litestream_pid, :log_writer
+  attr_reader :litestream_pid, :log_writer
 
   def start(launcher)
     @log_writer = launcher.log_writer
-    @puma_pid = $$
 
     launcher.events.on_booted do
-      @litestream_pid = fork do
-        Thread.new { monitor_puma }
-        Litestream::Commands.replicate(async: true)
-      end
+      @litestream_pid = Litestream::Commands.replicate(async: true)
 
       in_background do
         monitor_litestream
@@ -33,10 +29,6 @@ Puma::Plugin.create do
   rescue Errno::ECHILD, Errno::ESRCH
   end
 
-  def monitor_puma
-    monitor(:puma_dead?, "Detected Puma has gone away, stopping Litestream...")
-  end
-
   def monitor_litestream
     monitor(:litestream_dead?, "Detected Litestream has gone away, stopping Puma...")
   end
@@ -59,10 +51,6 @@ Puma::Plugin.create do
     true
   end
 
-  def puma_dead?
-    Process.ppid != puma_pid
-  end
-
   def log(...)
     log_writer.log(...)
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: litestream
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.15.0
 platform: ruby
 authors:
 - Stephen Margheim
 bindir: exe
 cert_chain: []
-date: 2025-06-13 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3
@@ -23,6 +23,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  force_ruby_platform: false
 - !ruby/object:Gem::Dependency
   name: actionpack
   requirement: !ruby/object:Gem::Requirement
@@ -37,6 +38,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '7.0'
+  force_ruby_platform: false
 - !ruby/object:Gem::Dependency
   name: actionview
   requirement: !ruby/object:Gem::Requirement
@@ -51,6 +53,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '7.0'
+  force_ruby_platform: false
 - !ruby/object:Gem::Dependency
   name: activejob
   requirement: !ruby/object:Gem::Requirement
@@ -65,6 +68,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '7.0'
+  force_ruby_platform: false
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -79,6 +83,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '7.0'
+  force_ruby_platform: false
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -93,6 +98,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '7.0'
+  force_ruby_platform: false
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -172,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.3
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Integrate Litestream with the RubyGems infrastructure.
 test_files: []