pgdexter 0.5.6 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd90908659094245f432d2a21115b35133f47885029e08daecf2c2b2ff74d016
-  data.tar.gz: f11515b062b235a28543afea6b927133a2caf538c323f09b9ebc7c69fab5a6fa
+  metadata.gz: 234a20833730445c80e03b5d789f174f9f50e9fb443614938912aaa1f74a7bd8
+  data.tar.gz: b286dd2c8735f17cd943c71405a2863699d4540487f5f1c730f5c19e6337da79
 SHA512:
-  metadata.gz: 89f629b0e98dc06c407a36f495cb5f2d78622c9d148a810defadf728dabd229609e86ff7afd1b32af6db0db1c69239da6e6cd0a90f6ecaf8954a2432b0d5b9ea
-  data.tar.gz: 228655a66e845bcffa368914cecea70c244ef7c5fad5b8625628c600a4494b1cfad8efaf711826cf2353113fcf198feeb9d0bda16e2bd7ba29e7a723218d57d9
+  metadata.gz: 6464fe3a0d24567a1759272ca8ab82b6462c88c5221f10a7784a0fdf41234268896b0368fbd20e442ce8f7a307d917c6779f484006c1c4f7971be07a8ba26de1
+  data.tar.gz: e3b0e7abc058f8f250e431d8b332e761a83bbc737b782449804a663f251dd7f33dc3213cc247dfb6be06b636c1d166af54803c13d7541dd9a8479baecaa86e8c

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.6.0 (2025-06-01)
+
+- Added Linux packages for Ubuntu 24.04 and Debian 12
+- Fixed error with hypothetical index limit
+- Dropped support for Linux package for Ubuntu 20.04
+- Dropped support for Ruby < 3
+- Dropped support for Postgres < 13
+
 ## 0.5.6 (2025-02-01)
 
 - Updated pg_query

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2017-2024 Andrew Kane
+Copyright (c) 2017-2025 Andrew Kane
 
 MIT License
 

data/README.md

@@ -91,7 +91,7 @@ Dexter can collect queries from a number of sources.
 
 - [Query stats](#query-stats)
 - [Live queries](#live-queries)
-- [Log files](#log-file)
+- [Log files](#log-files)
 - [SQL files](#sql-files)
 
 ### Query Stats
@@ -138,7 +138,7 @@ For real-time indexing, pipe your logfile:
 tail -F -n +1 postgresql.log | dexter <connection-options> --stdin
 ```
 
-And pass `--input-format csvlog` or `--input-format jsonlog` if needed.
+And pass `--input-format csv` or `--input-format json` if needed.
 
 ### SQL Files
 
@@ -232,10 +232,10 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
 For Ubuntu and Debian, use:
 
 ```sh
-sudo apt-get install postgresql-server-dev-15
+sudo apt-get install postgresql-server-dev-16
 ```
 
-Note: Replace `15` with your Postgres server version
+Note: Replace `16` with your Postgres server version
 
 ## Additional Installation Methods
 
@@ -282,16 +282,6 @@ gem install specific_install
 gem specific_install https://github.com/ankane/dexter.git
 ```
 
-## Upgrade Notes
-
-### 0.5.0
-
-The `--stdin` option is now required to read queries from stdin.
-
-```sh
-tail -F -n +1 postgresql.log | dexter <connection-options> --stdin
-```
-
 ## Thanks
 
 This software wouldn’t be possible without [HypoPG](https://github.com/HypoPG/hypopg), which allows you to create hypothetical indexes, and [pg_query](https://github.com/lfittl/pg_query), which allows you to parse and fingerprint queries. A big thanks to Dalibo and Lukas Fittl respectively. Also, thanks to YugabyteDB for [this article](https://dev.to/yugabyte/explain-from-pgstatstatements-normalized-queries-how-to-always-get-the-generic-plan-in--5cfi) on how to explain normalized queries.

data/lib/dexter/client.rb

@@ -6,8 +6,8 @@ module Dexter
     attr_reader :arguments, :options
 
     def self.start
-      Dexter::Client.new(ARGV).perform
-    rescue Dexter::Abort, PG::UndefinedFile, PG::FeatureNotSupported => e
+      Client.new(ARGV).perform
+    rescue Error => e
       abort colorize(e.message.strip, :red)
     end
 
@@ -19,33 +19,44 @@ module Dexter
       STDOUT.sync = true
       STDERR.sync = true
 
-      if options[:statement]
-        query = Query.new(options[:statement])
-        Indexer.new(options).process_queries([query])
-      elsif options[:pg_stat_statements]
-        # TODO support streaming option
-        Indexer.new(options).process_stat_statements
-      elsif options[:pg_stat_activity]
-        Processor.new(:pg_stat_activity, options).perform
-      elsif arguments.any?
-        ARGV.replace(arguments)
-        if !options[:input_format]
-          ext = ARGV.map { |v| File.extname(v) }.uniq
-          options[:input_format] = ext.first[1..-1] if ext.size == 1
+      connection = Connection.new(**options.slice(:dbname, :host, :port, :username, :log_sql))
+      connection.setup(options[:enable_hypopg])
+
+      source =
+        if options[:statement]
+          # TODO raise error for --interval, --min-calls, --min-time
+          StatementSource.new(options[:statement])
+        elsif options[:pg_stat_statements]
+          # TODO support streaming option
+          PgStatStatementsSource.new(connection)
+        elsif options[:pg_stat_activity]
+          PgStatActivitySource.new(connection)
+        elsif arguments.any?
+          ARGV.replace(arguments)
+          if !options[:input_format]
+            ext = ARGV.map { |v| File.extname(v) }.uniq
+            options[:input_format] = ext.first[1..-1] if ext.size == 1
+          end
+          LogSource.new(ARGF, options[:input_format])
+        elsif options[:stdin]
+          LogSource.new(STDIN, options[:input_format])
+        else
+          raise Error, "Specify a source of queries: --pg-stat-statements, --pg-stat-activity, --stdin, or a path"
         end
-        Processor.new(ARGF, options).perform
-      elsif options[:stdin]
-        Processor.new(STDIN, options).perform
-      else
-        raise Dexter::Abort, "Specify a source of queries: --pg-stat-statements, --pg-stat-activity, --stdin, or a path"
-      end
+
+      collector = Collector.new(**options.slice(:min_time, :min_calls))
+
+      indexer = Indexer.new(connection: connection, **options)
+
+      Processor.new(source, collector, indexer, **options.slice(:interval)).perform
     end
 
     def parse_args(args)
       opts = Slop.parse(args) do |o|
-        o.banner = %(Usage:
-    dexter [options])
-        o.separator ""
+        o.banner = <<~BANNER
+          Usage:
+              dexter [options]
+        BANNER
 
         o.separator "Input options:"
         o.string "--input-format", "input format"
@@ -64,8 +75,7 @@ module Dexter
 
         o.separator "Processing options:"
         o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
-        # TODO change to integer in 0.6.0
-        o.float "--min-calls", "only process queries that have been called a certain number of times", default: 0
+        o.integer "--min-calls", "only process queries that have been called a certain number of times", default: 0
         o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
         o.separator ""
 
@@ -75,6 +85,7 @@ module Dexter
         o.boolean "--enable-hypopg", "enable the HypoPG extension", default: false
         o.array "--exclude", "prevent specific tables from being indexed"
         o.string "--include", "only include specific tables"
+        o.integer "--min-cost", default: 100, help: false
         o.integer "--min-cost-savings-pct", default: 50, help: false
         o.string "--tablespace", "tablespace to create indexes"
         o.separator ""
@@ -103,11 +114,17 @@ module Dexter
 
       # TODO don't use global var
       $log_level = options[:log_level].to_s.downcase
-      raise Dexter::Abort, "Unknown log level" unless ["error", "info", "debug", "debug2", "debug3"].include?($log_level)
+      unless ["error", "info", "debug", "debug2", "debug3"].include?($log_level)
+        raise Error, "Unknown log level"
+      end
+
+      unless [nil, "csv", "json", "sql"].include?(options[:input_format])
+        raise Error, "Unknown input format"
+      end
 
       [arguments, options]
     rescue Slop::Error => e
-      raise Dexter::Abort, e.message
+      raise Error, e.message
     end
   end
 end

data/lib/dexter/collector.rb

@@ -1,22 +1,17 @@
 module Dexter
   class Collector
-    def initialize(options = {})
+    def initialize(min_time:, min_calls:)
       @top_queries = {}
       @new_queries = Set.new
       @mutex = Mutex.new
-      @min_time = options[:min_time] * 60000 # convert minutes to ms
-      @min_calls = options[:min_calls]
+      @min_time = min_time * 60000 # convert minutes to ms
+      @min_calls = min_calls
     end
 
-    def add(query, total_time, calls = 1)
-      fingerprint =
-        begin
-          PgQuery.fingerprint(query)
-        rescue PgQuery::ParseError
-          # do nothing
-        end
-
-      return unless fingerprint
+    def add(query, total_time, calls = 1, keep_all = false)
+      fingerprint = PgQuery.fingerprint(query) rescue nil
+      fingerprint ||= "unknown" if keep_all
+      return if fingerprint.nil?
 
       @top_queries[fingerprint] ||= {calls: 0, total_time: 0}
       @top_queries[fingerprint][:calls] += calls
@@ -36,16 +31,13 @@ module Dexter
       end
 
       queries = []
-      @top_queries.each do |k, v|
-        if new_queries.include?(k) && v[:total_time] >= @min_time && v[:calls] >= @min_calls
-          query = Query.new(v[:query], k)
-          query.total_time = v[:total_time]
-          query.calls = v[:calls]
-          queries << query
+      @top_queries.each do |fingerprint, query|
+        if new_queries.include?(fingerprint) && query[:total_time] >= @min_time && query[:calls] >= @min_calls
+          queries << Query.new(query[:query], fingerprint, total_time: query[:total_time], calls: query[:calls])
         end
       end
 
-      queries.sort_by(&:fingerprint)
+      queries
     end
   end
 end

data/lib/dexter/column_resolver.rb

@@ -0,0 +1,74 @@
+module Dexter
+  class ColumnResolver
+    include Logging
+
+    def initialize(connection, queries, log_level:)
+      @connection = connection
+      @queries = queries
+      @log_level = log_level
+    end
+
+    def perform
+      tables = Set.new(@queries.flat_map(&:candidate_tables))
+      columns = tables.any? ? self.columns(tables) : []
+      columns_by_table = columns.group_by { |c| c[:table] }.transform_values { |v| v.to_h { |c| [c[:column], c] } }
+      columns_by_table.default = {}
+
+      @queries.each do |query|
+        log "Finding columns: #{query.statement}" if @log_level == "debug3"
+        columns = Set.new
+        begin
+          find_columns(query.tree).each do |col|
+            last_col = col["fields"].last
+            if last_col["String"]
+              columns << last_col["String"]["sval"]
+            end
+          end
+        rescue JSON::NestingError
+          if @log_level.start_with?("debug")
+            log colorize("ERROR: Cannot get columns", :red)
+          end
+        end
+
+        possible_columns = []
+        columns.each do |column|
+          query.candidate_tables.each do |table|
+            resolved = columns_by_table.dig(table, column)
+            possible_columns << resolved if resolved
+          end
+        end
+        # use all columns in tables from views (not ideal)
+        (query.tables_from_views & query.candidate_tables).each do |table|
+          possible_columns.concat(columns_by_table[table].values)
+        end
+        query.columns = possible_columns.uniq
+      end
+    end
+
+    private
+
+    def find_columns(plan)
+      plan = JSON.parse(plan.to_json, max_nesting: 1000)
+      Indexer.find_by_key(plan, "ColumnRef")
+    end
+
+    def columns(tables)
+      query = <<~SQL
+        SELECT
+          s.nspname || '.' || t.relname AS table_name,
+          a.attname AS column_name,
+          pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type
+        FROM pg_attribute a
+          JOIN pg_class t on a.attrelid = t.oid
+          JOIN pg_namespace s on t.relnamespace = s.oid
+        WHERE a.attnum > 0
+          AND NOT a.attisdropped
+          AND s.nspname || '.' || t.relname IN (#{tables.size.times.map { |i| "$#{i + 1}" }.join(", ")})
+        ORDER BY
+          1, 2
+      SQL
+      columns = @connection.execute(query, params: tables.to_a)
+      columns.map { |v| {table: v["table_name"], column: v["column_name"], type: v["data_type"]} }
+    end
+  end
+end

data/lib/dexter/connection.rb

@@ -0,0 +1,92 @@
+module Dexter
+  class Connection
+    include Logging
+
+    def initialize(dbname:, host:, port:, username:, log_sql:)
+      @dbname = dbname
+      @host = host
+      @port = port
+      @username = username
+      @log_sql = log_sql
+      @mutex = Mutex.new
+    end
+
+    def setup(enable_hypopg)
+      if server_version_num < 130000
+        raise Error, "This version of Dexter requires Postgres 13+"
+      end
+
+      check_extension(enable_hypopg)
+
+      execute("SET lock_timeout = '5s'")
+    end
+
+    def execute(query, pretty: true, params: [], use_exec: false)
+      # use exec_params instead of exec when possible for security
+      #
+      # Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
+      # (There can be semicolons in it, but not more than one nonempty command.)
+      # This is a limitation of the underlying protocol, but has some usefulness
+      # as an extra defense against SQL-injection attacks.
+      # https://www.postgresql.org/docs/current/static/libpq-exec.html
+      query = squish(query) if pretty
+      log colorize("[sql] #{query}#{params.any? ? " /*#{params.to_json}*/" : ""}", :cyan) if @log_sql
+
+      @mutex.synchronize do
+        if use_exec
+          conn.exec("#{query} /*dexter*/").to_a
+        else
+          conn.exec_params("#{query} /*dexter*/", params).to_a
+        end
+      end
+    end
+
+    def server_version_num
+      @server_version_num ||= execute("SHOW server_version_num").first["server_version_num"].to_i
+    end
+
+    private
+
+    def check_extension(enable_hypopg)
+      extension = execute("SELECT installed_version FROM pg_available_extensions WHERE name = 'hypopg'").first
+
+      if extension.nil?
+        raise Error, "Install HypoPG first: https://github.com/ankane/dexter#installation"
+      end
+
+      if extension["installed_version"].nil?
+        if enable_hypopg
+          execute("CREATE EXTENSION hypopg")
+        else
+          raise Error, "Run `CREATE EXTENSION hypopg` or pass --enable-hypopg"
+        end
+      end
+    end
+
+    def conn
+      @conn ||= begin
+        # set connect timeout if none set
+        ENV["PGCONNECT_TIMEOUT"] ||= "3"
+
+        if @dbname.to_s.start_with?("postgres://", "postgresql://")
+          config = @dbname
+        else
+          config = {
+            host: @host,
+            port: @port,
+            dbname: @dbname,
+            user: @username
+          }.reject { |_, value| value.to_s.empty? }
+          config = config[:dbname] if config.keys == [:dbname] && config[:dbname].include?("=")
+        end
+        PG::Connection.new(config)
+      end
+    rescue PG::ConnectionBad => e
+      raise Error, e.message
+    end
+
+    def squish(str)
+      str.to_s.gsub(/\s+/, " ").strip
+    end
+  end
+end