sqlitesweep 0.1.0

data/lib/sqlitesweep/query/local.rb

@@ -0,0 +1,32 @@
+require "sqlite3"
+
+module SQLiteSweep
+  module Query
+    # Executes SQL queries against local SQLite databases using the sqlite3 gem.
+    #
+    # Opens each database in readonly mode, executes the query, and returns
+    # results as an array of hashes. The database connection is always closed
+    # after the query, even if an error occurs.
+    #
+    # @example
+    #   local = Query::Local.new(config)
+    #   result = local.execute(DatabaseURI.new("/tmp/test.sqlite3"))
+    #   result.rows  # => [{"count(*)" => 42}]
+    #
+    class Local < Base
+      # @param uri [DatabaseURI] A local database URI.
+      # @return [Result] The query result.
+      # @raise [QueryError] If the database can't be opened or the query fails.
+      def execute(uri)
+        db = SQLite3::Database.new(uri.path, readonly: true)
+        db.results_as_hash = true
+        rows = db.execute(@config.query)
+        Result.new(rows: rows, source: uri)
+      rescue SQLite3::Exception => e
+        raise QueryError, "Local query failed on #{uri}: #{e.message}"
+      ensure
+        db&.close
+      end
+    end
+  end
+end

data/lib/sqlitesweep/query/remote.rb

@@ -0,0 +1,90 @@
+require "json"
+require "shellwords"
+
+module SQLiteSweep
+  module Query
+    # Executes batched SQL queries on remote hosts via SSH.
+    #
+    # Instead of one SSH round-trip per database, this runs a single SSH command
+    # that queries multiple databases sequentially on the remote host. The remote
+    # command iterates over database paths, running sqlite3 -json on each one,
+    # and outputs tab-delimited lines:
+    #
+    #   /path/to/db1.sqlite3\t[{"count(*)":42}]
+    #   /path/to/db2.sqlite3\t[{"count(*)":17}]
+    #
+    # The output is parsed back into individual Result objects, one per database.
+    #
+    # @example
+    #   remote = Query::Remote.new(config, ssh_manager)
+    #   results = remote.execute_batch([uri1, uri2, uri3])
+    #   results.each { |r| puts "#{r.source}: #{r.rows}" }
+    #
+    class Remote < Base
+      def initialize(config, ssh_manager)
+        super(config)
+        @ssh_manager = ssh_manager
+      end
+
+      # Queries multiple databases on the same remote host in a single SSH call.
+      #
+      # @param uris [Array<DatabaseURI>] URIs for databases on the same host.
+      #   All URIs must share the same host_key.
+      # @return [Array<Result>] One Result per successfully queried database.
+      # @raise [QueryError] If the SSH command fails or JSON parsing fails.
+      def execute_batch(uris)
+        return [] if uris.empty?
+
+        host_key = uris.first.host_key
+        paths = uris.map(&:path)
+
+        # Escape single quotes in the SQL for safe embedding in the shell command
+        sql = @config.query.gsub("'", "'\\''")
+
+        # Build a remote script that queries each database and outputs
+        # tab-delimited results: <db_path>\t<json_result>
+        remote_script = paths.map { |path|
+          escaped_path = Shellwords.shellescape(path)
+          "printf '%s\\t' #{escaped_path}; sqlite3 -json #{escaped_path} '#{sql}'; echo"
+        }.join("; ")
+
+        output = @ssh_manager.run(host_key, remote_script)
+        parse_batch_output(output, uris)
+      end
+
+      private
+
+      # Parses the tab-delimited batch output back into individual Results.
+      #
+      # Each line is expected to be: <db_path>\t<json_array>
+      # Lines that don't match this format are silently skipped.
+      def parse_batch_output(output, uris)
+        results = []
+        uri_by_path = uris.each_with_object({}) { |u, h| h[u.path] = u }
+
+        output.each_line do |line|
+          line = line.strip
+          next if line.empty?
+
+          tab_idx = line.index("\t")
+          next unless tab_idx
+
+          db_path = line[0...tab_idx]
+          json_str = line[(tab_idx + 1)..]
+
+          uri = uri_by_path[db_path]
+          next unless uri
+
+          begin
+            rows = json_str.empty? ? [] : JSON.parse(json_str)
+            results << Result.new(rows: rows, source: uri)
+          rescue JSON::ParserError => e
+            raise QueryError, "Failed to parse JSON from #{db_path}: #{e.message}"
+          end
+        end
+
+        results
+      end
+    end
+  end
+end

data/lib/sqlitesweep/result.rb

@@ -0,0 +1,17 @@
+module SQLiteSweep
+  # Represents the result of querying a single database.
+  #
+  # @!attribute [r] rows
+  #   @return [Array<Hash>] Array of row hashes returned by the SQL query.
+  #     For example, [{"count(*)" => 42}].
+  #
+  # @!attribute [r] source
+  #   @return [String] The URI string of the database that produced this result.
+  #     Used to annotate list output with _source provenance.
+  #
+  Result = Data.define(:rows, :source) do
+    def initialize(rows:, source:)
+      super(rows: rows, source: source.to_s)
+    end
+  end
+end

data/lib/sqlitesweep/result_file.rb

@@ -0,0 +1,53 @@
+require "json"
+require "tempfile"
+
+module SQLiteSweep
+  # Thread-safe JSONL file writer for the :list action.
+  #
+  # Instead of holding all results in memory (which would be problematic when
+  # sweeping millions of databases), results are streamed to a JSONL file as
+  # they arrive. Each line is a JSON object with the query row data plus a
+  # "_source" field indicating which database it came from.
+  #
+  # The file persists after the process exits so external tools can consume it.
+  # The file path is printed to stdout as the final output of a :list sweep.
+  #
+  # @example Output file contents (one JSON object per line):
+  #   {"count(*)":42,"_source":"/data/tenant_1.sqlite3"}
+  #   {"count(*)":17,"_source":"ssh://deploy@web1/data/tenant_2.sqlite3"}
+  #
+  class ResultFile
+    # @return [String] Absolute path to the JSONL output file.
+    attr_reader :path
+
+    # @return [Integer] Total number of rows written so far.
+    attr_reader :row_count
+
+    def initialize
+      @path = File.join(Dir.tmpdir, "sqlitesweep_results_#{Process.pid}_#{Time.now.to_i}.jsonl")
+      @file = File.open(@path, "w")
+      @mutex = Mutex.new
+      @row_count = 0
+    end
+
+    # Writes all rows from a Result to the file. Each row becomes one JSON line,
+    # annotated with "_source" for provenance tracking.
+    #
+    # @param result [Result] The query result to write.
+    def write(result)
+      @mutex.synchronize do
+        result.rows.each do |row|
+          line = row.merge("_source" => result.source).to_json
+          @file.puts(line)
+          @row_count += 1
+        end
+        @file.flush
+      end
+    end
+
+    # Closes the file handle. The file itself remains on disk for consumption.
+    def close
+      @file.close unless @file.closed?
+    end
+  end
+end

data/lib/sqlitesweep/runner.rb

@@ -0,0 +1,93 @@
+module SQLiteSweep
+  # Main orchestrator. Wires all components together and runs the sweep.
+  #
+  # Flow:
+  #   1. Reads database URIs from SourceStream (which runs the -s command)
+  #   2. Local URIs are submitted directly to the WorkerPool for Query::Local
+  #   3. Remote URIs are sent to HostBatcher, which groups them by host and
+  #      flushes batches to the WorkerPool for Query::Remote
+  #   4. Results feed into the Aggregator (thread-safe sum/avg/list)
+  #   5. Display refreshes a live status line on stderr
+  #   6. Final aggregated result is printed to stdout
+  #
+  class Runner
+    def initialize(config)
+      @config = config
+      @shutting_down = false
+    end
+
+    # Executes the full sweep. Blocks until all databases are queried and
+    # the final result is printed to stdout.
+    def run
+      result_file = ResultFile.new if @config.action == :list
+      aggregator = Aggregator.new(@config.action, result_file: result_file)
+      display = Display.new(aggregator, live: @config.live)
+      pool = WorkerPool.new(@config.concurrency)
+      local_query = Query::Local.new(@config)
+      ssh_manager = SSH::ConnectionManager.new(@config)
+      batcher = HostBatcher.new(@config, pool, aggregator, display, ssh_manager)
+
+      setup_signal_handlers(pool, ssh_manager, display, batcher)
+
+      source = SourceStream.new(@config.source)
+      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      display.start(start_time)
+
+      # Main loop: read URIs and dispatch to the appropriate query path
+      source.each do |uri|
+        break if @shutting_down
+
+        if uri.local?
+          pool.submit do
+            begin
+              result = local_query.execute(uri)
+              aggregator.add(result)
+            rescue QueryError => e
+              aggregator.record_error
+              $stderr.puts "\n#{e.message}" unless @config.live
+            end
+            display.refresh
+          end
+        else
+          batcher.add(uri)
+        end
+      end
+
+      # Drain remaining batches that haven't hit batch_size yet
+      batcher.flush_all
+      pool.shutdown
+
+      display.finish
+      result_file&.close
+
+      output = aggregator.value
+      puts output
+    rescue SourceError => e
+      $stderr.puts "\nError: #{e.message}"
+      exit 1
+    ensure
+      ssh_manager&.shutdown
+    end
+
+    private
+
+    # Installs a SIGINT (Ctrl+C) handler for graceful shutdown.
+    # First Ctrl+C: cancel pending work, drain pool, clean up SSH.
+    # Second Ctrl+C: force exit immediately.
+    def setup_signal_handlers(pool, ssh_manager, display, batcher)
+      trap("INT") do
+        if @shutting_down
+          exit! 1
+        else
+          @shutting_down = true
+          $stderr.write "\nShutting down...\n"
+          batcher.cancel
+          pool.kill
+          display.finish
+          ssh_manager.shutdown
+          exit 1
+        end
+      end
+    end
+  end
+end

data/lib/sqlitesweep/source_stream.rb

@@ -0,0 +1,44 @@
+module SQLiteSweep
+  # Reads database URIs from a shell command, one per line.
+  #
+  # The command is run via IO.popen and its stdout is read line-by-line.
+  # Each non-empty line is parsed into a DatabaseURI. Blank lines are skipped.
+  #
+  # The command can be anything: a simple `cat`, a `rails runner` invocation,
+  # a script that queries an API, etc. Whatever it is, it must output one
+  # database URI per line to stdout.
+  #
+  # @example
+  #   stream = SourceStream.new("cat /tmp/uris.txt")
+  #   stream.each { |uri| puts uri.path }
+  #
+  # @example with Rails
+  #   stream = SourceStream.new('rails runner "Tenant.find_each { |t| puts t.db_path }"')
+  #
+  class SourceStream
+    def initialize(command)
+      @command = command
+    end
+
+    # Yields each DatabaseURI from the source command's output.
+    # Returns an Enumerator if no block is given.
+    #
+    # @yieldparam uri [DatabaseURI] A parsed database URI.
+    # @raise [SourceError] If the command exits with a non-zero status.
+    def each(&block)
+      return enum_for(:each) unless block_given?
+
+      IO.popen(@command, "r") do |io|
+        io.each_line do |line|
+          line = line.strip
+          next if line.empty?
+          yield DatabaseURI.new(line)
+        end
+      end
+
+      unless $?.success?
+        raise SourceError, "Source command exited with status #{$?.exitstatus}: #{@command}"
+      end
+    end
+  end
+end

data/lib/sqlitesweep/ssh/connection_manager.rb

@@ -0,0 +1,142 @@
+require "concurrent"
+require "open3"
+require "fileutils"
+
+module SQLiteSweep
+  module SSH
+    # Manages SSH ControlMaster connections for multiplexed remote access.
+    #
+    # Why ControlMaster instead of net-ssh?
+    #   - net-ssh is not thread-safe for concurrent operations on a single session
+    #   - System ssh with ControlMaster handles multiplexing natively
+    #   - SSH agent, config files, and known_hosts all work automatically
+    #
+    # How it works:
+    #   1. On first query to a host, establishes a background master connection
+    #      (ssh -N -f with ControlMaster=yes) that stays alive via ControlPersist
+    #   2. Subsequent SSH commands to the same host multiplex over the existing
+    #      master via ControlPath — no new TCP/auth handshake needed
+    #   3. A semaphore enforces --max-ssh to cap total master connections
+    #   4. On shutdown, sends "ssh -O exit" to each master and cleans up sockets
+    #
+    # Socket files live in /tmp/sqlitesweep_ssh_<pid>/ and are cleaned up
+    # on shutdown.
+    #
+    class ConnectionManager
+      def initialize(config)
+        @config = config
+        @socket_dir = File.join("/tmp", "sqlitesweep_ssh_#{Process.pid}")
+        FileUtils.mkdir_p(@socket_dir)
+        @semaphore = Concurrent::Semaphore.new(config.max_ssh)
+        @masters = Concurrent::Map.new  # host_key => true (thread-safe set)
+        @mutex = Mutex.new
+      end
+
+      # Returns the Unix socket path for a given host's ControlMaster.
+      #
+      # @param host_key [String] The SSH destination (e.g. "deploy@web1").
+      # @return [String] Path to the socket file.
+      def socket_path(host_key)
+        File.join(@socket_dir, host_key.gsub("/", "_"))
+      end
+
+      # Ensures a ControlMaster connection exists for the given host.
+      # If one already exists, returns immediately. Otherwise, acquires a
+      # semaphore permit and establishes a new master.
+      #
+      # Uses BatchMode=yes to prevent password prompts (fails fast if
+      # key-based auth isn't configured). StrictHostKeyChecking=accept-new
+      # auto-accepts new hosts but rejects changed keys.
+      #
+      # @param host_key [String] The SSH destination (e.g. "deploy@web1").
+      # @raise [SSHError] If the master connection fails to establish.
+      def ensure_master(host_key)
+        return if @masters[host_key]
+
+        @semaphore.acquire
+        begin
+          # Double-check after acquiring semaphore (another thread may have
+          # established the master while we were waiting)
+          return if @masters[host_key]
+
+          socket = socket_path(host_key)
+          cmd = [
+            "ssh",
+            "-o", "ControlMaster=yes",
+            "-o", "ControlPath=#{socket}",
+            "-o", "ControlPersist=120",
+            "-o", "BatchMode=yes",
+            "-o", "StrictHostKeyChecking=accept-new",
+            "-o", "ConnectTimeout=#{@config.ssh_timeout}",
+            "-N", "-f",  # No command, go to background
+            host_key
+          ]
+
+          _out, err, status = Open3.capture3(*cmd)
+          unless status.success?
+            @semaphore.release
+            raise SSHError, "Failed to establish SSH master to #{host_key}: #{err.strip}"
+          end
+
+          @masters[host_key] = true
+        rescue SSHError
+          raise
+        rescue => e
+          @semaphore.release
+          raise SSHError, "SSH connection error to #{host_key}: #{e.message}"
+        end
+      end
+
+      # Runs a command on a remote host over the multiplexed SSH connection.
+      # Ensures a ControlMaster exists first, then executes via ControlPath.
+      #
+      # @param host_key [String] The SSH destination.
+      # @param remote_command [String] The shell command to run remotely.
+      # @param timeout [Integer, nil] Override timeout in seconds (defaults to config.query_timeout).
+      # @return [String] The command's stdout.
+      # @raise [QueryError] If the remote command exits non-zero.
+      # @raise [TimeoutError] If the command exceeds the timeout.
+      def run(host_key, remote_command, timeout: nil)
+        ensure_master(host_key)
+
+        socket = socket_path(host_key)
+        cmd = [
+          "ssh",
+          "-o", "ControlPath=#{socket}",
+          "-o", "BatchMode=yes",
+          host_key,
+          remote_command
+        ]
+
+        timeout ||= @config.query_timeout
+        out, err, status = execute_with_timeout(cmd, timeout)
+
+        unless status.success?
+          raise QueryError, "Remote command failed on #{host_key}: #{err.strip}"
+        end
+
+        out
+      end
+
+      # Shuts down all ControlMaster connections and cleans up socket files.
+      # Sends "ssh -O exit" to each master to gracefully close it.
+      def shutdown
+        @masters.each_key do |host_key|
+          socket = socket_path(host_key)
+          system("ssh", "-o", "ControlPath=#{socket}", "-O", "exit", host_key,
+                 out: File::NULL, err: File::NULL)
+        end
+        @masters.clear
+        FileUtils.rm_rf(@socket_dir) if Dir.exist?(@socket_dir)
+      end
+
+      private
+
+      def execute_with_timeout(cmd, timeout)
+        Open3.capture3(*cmd, timeout: timeout)
+      rescue Timeout::Error
+        raise TimeoutError, "Command timed out after #{timeout}s: #{cmd.last(1).first}"
+      end
+    end
+  end
+end

data/lib/sqlitesweep/version.rb

@@ -0,0 +1,3 @@
+module SQLiteSweep
+  VERSION = "0.1.0"
+end

data/lib/sqlitesweep/worker_pool.rb

@@ -0,0 +1,45 @@
+require "concurrent"
+
+module SQLiteSweep
+  # Thread pool wrapper around Concurrent::FixedThreadPool.
+  #
+  # Uses a fixed number of threads for IO-bound SSH and SQLite work. The
+  # :caller_runs fallback policy provides natural back-pressure: when all
+  # threads are busy, the submitting thread runs the work itself rather
+  # than queuing unboundedly.
+  #
+  # @example
+  #   pool = WorkerPool.new(8)
+  #   pool.submit { query_database(uri) }
+  #   pool.shutdown  # waits for all work to complete
+  #
+  class WorkerPool
+    # @param size [Integer] Number of worker threads.
+    def initialize(size)
+      @pool = Concurrent::FixedThreadPool.new(size, fallback_policy: :caller_runs)
+    end
+
+    # Submits a block to be executed by a worker thread.
+    # If all workers are busy, the block runs on the calling thread
+    # (back-pressure via :caller_runs).
+    #
+    # @yieldreturn [void]
+    def submit(&block)
+      @pool.post(&block)
+    end
+
+    # Gracefully shuts down the pool: stops accepting new work and waits
+    # for all submitted work to finish.
+    #
+    # @param timeout [Integer] Max seconds to wait for completion.
+    def shutdown(timeout = 60)
+      @pool.shutdown
+      @pool.wait_for_termination(timeout)
+    end
+
+    # Immediately kills all worker threads. Used for forced shutdown (e.g. Ctrl+C).
+    def kill
+      @pool.kill
+    end
+  end
+end

data/lib/sqlitesweep.rb

@@ -0,0 +1,53 @@
+# SQLiteSweep - Query millions of SQLite databases across remote hosts via SSH.
+#
+# Designed for multi-tenant applications where each tenant has their own SQLite
+# database. Reads database URIs from a shell command, queries each one in parallel,
+# and aggregates results in real-time.
+#
+# Architecture overview:
+#
+#   SourceStream   reads URIs from a shell command (IO.popen)
+#       |
+#   Runner         main orchestrator, dispatches URIs to the right path
+#       |
+#       +-- local URIs --> WorkerPool --> Query::Local (sqlite3 gem)
+#       |
+#       +-- remote URIs --> HostBatcher --> WorkerPool --> Query::Remote (ssh + sqlite3 CLI)
+#                               |
+#                       SSH::ConnectionManager (ControlMaster pooling)
+#       |
+#   Aggregator     thread-safe accumulator (sum / average / list)
+#       |
+#   Display        live ANSI progress on stderr
+#       |
+#   stdout         final result
+#
+# Usage:
+#   sqlitesweep -q "SELECT count(*) FROM users" -a sum -s "cat uris.txt"
+#
+require_relative "sqlitesweep/version"
+require_relative "sqlitesweep/errors"
+
+module SQLiteSweep
+  autoload :Config, "sqlitesweep/config"
+  autoload :CLI, "sqlitesweep/cli"
+  autoload :Runner, "sqlitesweep/runner"
+  autoload :SourceStream, "sqlitesweep/source_stream"
+  autoload :DatabaseURI, "sqlitesweep/database_uri"
+  autoload :Result, "sqlitesweep/result"
+  autoload :Aggregator, "sqlitesweep/aggregator"
+  autoload :WorkerPool, "sqlitesweep/worker_pool"
+  autoload :HostBatcher, "sqlitesweep/host_batcher"
+  autoload :Display, "sqlitesweep/display"
+  autoload :ResultFile, "sqlitesweep/result_file"
+
+  module Query
+    autoload :Base, "sqlitesweep/query/base"
+    autoload :Local, "sqlitesweep/query/local"
+    autoload :Remote, "sqlitesweep/query/remote"
+  end
+
+  module SSH
+    autoload :ConnectionManager, "sqlitesweep/ssh/connection_manager"
+  end
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: sqlitesweep
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Chris Maximin
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: CLI tool that queries SQLite databases across remote hosts via SSH, aggregating
+  results in real-time. Designed for multi-tenant apps where each tenant has their
+  own SQLite database.
+executables:
+- sqlitesweep
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- exe/sqlitesweep
+- lib/sqlitesweep.rb
+- lib/sqlitesweep/aggregator.rb
+- lib/sqlitesweep/cli.rb
+- lib/sqlitesweep/config.rb
+- lib/sqlitesweep/database_uri.rb
+- lib/sqlitesweep/display.rb
+- lib/sqlitesweep/errors.rb
+- lib/sqlitesweep/host_batcher.rb
+- lib/sqlitesweep/query/base.rb
+- lib/sqlitesweep/query/local.rb
+- lib/sqlitesweep/query/remote.rb
+- lib/sqlitesweep/result.rb
+- lib/sqlitesweep/result_file.rb
+- lib/sqlitesweep/runner.rb
+- lib/sqlitesweep/source_stream.rb
+- lib/sqlitesweep/ssh/connection_manager.rb
+- lib/sqlitesweep/version.rb
+- lib/sqlitesweep/worker_pool.rb
+homepage: https://github.com/chrismaximin/sqlitesweep
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '4.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.3
+specification_version: 4
+summary: Query millions of SQLite databases across remote hosts via SSH
+test_files: []