pgdexter 0.4.1 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 109808872c6a579e6ce82956d5b2b1da8c66af7c85fae3d5984367e374484eb2
-  data.tar.gz: 6fe0f25f316afd84c287ba2fefe5076846c8a572bf2cbee406843abf632ddb2d
+  metadata.gz: d2b2528c0ff8201ce676b1a4ca7ec11de52013bfc608fab1b574122a31dd5ae9
+  data.tar.gz: 570f24c171e02425a5c492068ccedf48e318194333d825cc384ce6cfeeb6c30a
 SHA512:
-  metadata.gz: c2e5b5d80869c5c90b234af5edbb6d0c0e6118a3dc7191c2d7169209c7ae1505329fb18ea5e3803b045a527dd20b2909a9b2067732dfa030bbce0a931fc88954
-  data.tar.gz: 03d139a0f6ac0e967ad4f98685a6d8e513eab0b2022c7a0f7c6d27eca0da9b6441c30480a19acacf5c5f5f0ec762a7243331c3ee752b071015800a98043cad3b
+  metadata.gz: e0975c42916c22b0d8f0b12f0144865370372771526ce32574a4602918aadd786b8ab39cb092ad28590c61c86820c51dcac4f7a41cc5930412868ce2a0590498
+  data.tar.gz: e91d272a0d6bc737634615118a1a33cbbf2ac2fe957aabb17edae9ae42fa17f596052e83a9caa80b71effa6b20316b12e551bd8aea7734648b3210bdb2bf00ec

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.4.3 (2023-03-26)
+
+- Added experimental `--log-table` option
+- Improved help
+- Require pg_query < 4
+
+## 0.4.2 (2023-01-29)
+
+- Fixed `--pg-stat-statements` option for Postgres 13+
+
 ## 0.4.1 (2022-10-15)
 
 - Added support for `json` format

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2017-2021 Andrew Kane
+Copyright (c) 2017-2023 Andrew Kane
 
 MIT License
 

data/README.md

@@ -110,14 +110,6 @@ or collect running queries with:
 dexter <connection-options> --pg-stat-activity
 ```
 
-or use the [pg_stat_statements](https://www.postgresql.org/docs/current/static/pgstatstatements.html) extension:
-
-```sh
-dexter <connection-options> --pg-stat-statements
-```
-
-> Note: Logs or running queries are highly preferred over pg_stat_statements, as pg_stat_statements often doesn’t store enough information to optimize queries.
-
 ### Collection Options
 
 To prevent one-off queries from being indexed, specify a minimum number of calls before a query is considered for indexing
@@ -190,11 +182,13 @@ dexter --log-sql --log-level debug2
 
 ## Hosted Postgres
 
-Some hosted providers like Amazon RDS and Heroku do not support the HypoPG extension, which Dexter needs to run. See [how to use Dexter](guides/Hosted-Postgres.md) in these cases. To request the extension:
+The `hypopg` extension, which Dexter needs to run, is available on [these providers](https://github.com/ankane/dexter/issues/44).
+
+For other providers, see [this guide](guides/Hosted-Postgres.md). To request a new extension:
 
 - Amazon RDS - follow the instructions on [this page](https://aws.amazon.com/rds/postgresql/faqs/)
-- Google Cloud SQL - star the [feature request](https://issuetracker.google.com/issues/69250435)
-- DigitalOcean Managed Databases - follow the instructions on [this page](https://docs.digitalocean.com/products/databases/postgresql/details/supported-extensions/#supported-extensions)
+- Google Cloud SQL - vote or comment on [this page](https://issuetracker.google.com/issues/69250435)
+- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/app-framework-services/p/support-hypopg-for-postgres)
 
 ## Additional Installation Methods
 
@@ -212,7 +206,7 @@ And run it with:
 docker run -ti ankane/dexter <connection-options>
 ```
 
-For databases on the host machine, use `host.docker.internal` as the hostname (on Linux, this requires Docker 20.04 and `--add-host=host.docker.internal:host-gateway`).
+For databases on the host machine, use `host.docker.internal` as the hostname (on Linux, this requires Docker 20.04+ and `--add-host=host.docker.internal:host-gateway`).
 
 ### Homebrew
 

data/lib/dexter/client.rb

@@ -27,6 +27,8 @@ module Dexter
         Indexer.new(options).process_stat_statements
       elsif options[:pg_stat_activity]
         Processor.new(:pg_stat_activity, options).perform
+      elsif options[:log_table]
+        Processor.new(:log_table, options).perform
       elsif arguments.any?
         ARGV.replace(arguments)
         Processor.new(ARGF, options).perform
@@ -40,23 +42,45 @@ module Dexter
         o.banner = %(Usage:
     dexter [options])
         o.separator ""
-        o.separator "Options:"
+
+        o.separator "Input options:"
+        o.string "--input-format", "input format", default: "stderr"
+        o.string "--log-table", "log table (experimental)"
+        o.boolean "--pg-stat-activity", "use pg_stat_activity", default: false, help: false
+        o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
+        o.string "-s", "--statement", "process a single statement"
+        o.separator ""
+
+        o.separator "Connection options:"
+        o.string "-d", "--dbname", "database name"
+        o.string "-h", "--host", "database host"
+        o.integer "-p", "--port", "database port"
+        o.string "-U", "--username", "database user"
+        o.separator ""
+
+        o.separator "Processing options:"
+        o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
+        o.float "--min-calls", "only process queries that have been called a certain number of times", default: 0
+        o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
+        o.boolean "--once", "run once", default: false, help: false
+        o.separator ""
+
+        o.separator "Indexing options:"
         o.boolean "--analyze", "analyze tables that haven't been analyzed in the past hour", default: false
         o.boolean "--create", "create indexes", default: false
         o.array "--exclude", "prevent specific tables from being indexed"
         o.string "--include", "only include specific tables"
-        o.string "--input-format", "input format", default: "stderr"
-        o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
+        o.integer "--min-cost-savings-pct", default: 50, help: false
+        o.string "--tablespace", "tablespace to create indexes"
+        o.separator ""
+
+        o.separator "Logging options:"
         o.boolean "--log-explain", "log explain", default: false, help: false
         o.string "--log-level", "log level", default: "info"
         o.boolean "--log-sql", "log sql", default: false
-        o.float "--min-calls", "only process queries that have been called a certain number of times", default: 0
-        o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
-        o.integer "--min-cost-savings-pct", default: 50, help: false
-        o.boolean "--pg-stat-activity", "use pg_stat_activity", default: false, help: false
-        o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
-        o.string "-s", "--statement", "process a single statement"
-        o.string "--tablespace", "tablespace to create indexes"
+        o.separator ""
+
+        o.separator "Other options:"
         o.on "-v", "--version", "print the version" do
           log Dexter::VERSION
           exit
@@ -65,12 +89,6 @@ module Dexter
           log o
           exit
         end
-        o.separator ""
-        o.separator "Connection options:"
-        o.string "-d", "--dbname", "database name"
-        o.string "-h", "--host", "database host"
-        o.integer "-p", "--port", "database port"
-        o.string "-U", "--username", "database user"
       end
 
       arguments = opts.arguments

data/lib/dexter/csv_log_parser.rb

@@ -1,22 +1,24 @@
-require "csv"
-
 module Dexter
   class CsvLogParser < LogParser
     FIRST_LINE_REGEX = /\A.+/
 
     def perform
       CSV.new(@logfile.to_io).each do |row|
-        if (m = REGEX.match(row[13]))
-          # replace first line with match
-          # needed for multiline queries
-          active_line = row[13].sub(FIRST_LINE_REGEX, m[3])
-
-          add_parameters(active_line, row[14]) if row[14]
-          process_entry(active_line, m[1].to_f)
-        end
+        process_csv_row(row[13], row[14])
       end
     rescue CSV::MalformedCSVError => e
       raise Dexter::Abort, "ERROR: #{e.message}"
     end
+
+    def process_csv_row(message, detail)
+      if (m = REGEX.match(message))
+        # replace first line with match
+        # needed for multiline queries
+        active_line = message.sub(FIRST_LINE_REGEX, m[3])
+
+        add_parameters(active_line, detail) if detail
+        process_entry(active_line, m[1].to_f)
+      end
+    end
   end
 end

data/lib/dexter/csv_log_table_parser.rb

@@ -0,0 +1,20 @@
+module Dexter
+  class CsvLogTableParser < CsvLogParser
+    def perform
+      last_log_time = Time.at(0).iso8601(3)
+
+      loop do
+        @logfile.csvlog_activity(last_log_time).each do |row|
+          process_csv_row(row["message"], row["detail"])
+          last_log_time = row["log_time"]
+        end
+
+        break
+
+        # possibly enable later
+        # break if once
+        # sleep(1)
+      end
+    end
+  end
+end

data/lib/dexter/indexer.rb

@@ -14,6 +14,7 @@ module Dexter
       @min_calls = options[:min_calls] || 0
       @analyze = options[:analyze]
       @min_cost_savings_pct = options[:min_cost_savings_pct].to_i
+      @log_table = options[:log_table]
       @options = options
       @mutex = Mutex.new
 
@@ -28,7 +29,7 @@ module Dexter
     end
 
     def stat_activity
-      execute <<-SQL
+      execute <<~SQL
         SELECT
           pid || ':' || COALESCE(query_start, xact_start) AS id,
           query,
@@ -44,6 +45,36 @@ module Dexter
       SQL
     end
 
+    # works with
+    # file_fdw: https://www.postgresql.org/docs/current/file-fdw.html
+    # log_fdw: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.Extensions.foreign-data-wrappers.html
+    def csvlog_activity(last_log_time)
+      query = <<~SQL
+        SELECT
+          log_time,
+          message,
+          detail
+        FROM
+          #{conn.quote_ident(@log_table)}
+        WHERE
+          log_time >= \$1
+      SQL
+      execute(query, params: [last_log_time])
+    end
+
+    # works with
+    # file_fdw: https://www.postgresql.org/docs/current/file-fdw.html
+    # log_fdw: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.Extensions.foreign-data-wrappers.html
+    def stderr_activity
+      query = <<~SQL
+        SELECT
+          log_entry
+        FROM
+          #{conn.quote_ident(@log_table)}
+      SQL
+      execute(query)
+    end
+
     def process_queries(queries)
       # reset hypothetical indexes
       reset_hypothetical_indexes
@@ -141,7 +172,7 @@ module Dexter
     def analyze_tables(tables)
       tables = tables.to_a.sort
 
-      analyze_stats = execute <<-SQL
+      analyze_stats = execute <<~SQL
         SELECT
           schemaname || '.' || relname AS table,
           last_analyze,
@@ -529,7 +560,7 @@ module Dexter
       raise Dexter::Abort, e.message
     end
 
-    def execute(query, pretty: true)
+    def execute(query, pretty: true, params: [])
       # use exec_params instead of exec for security
       #
       # Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
@@ -541,7 +572,7 @@ module Dexter
       log colorize("[sql] #{query}", :cyan) if @log_sql
 
       @mutex.synchronize do
-        conn.exec_params(query, []).to_a
+        conn.exec_params(query, params).to_a
       end
     end
 
@@ -565,7 +596,7 @@ module Dexter
     end
 
     def database_tables
-      result = execute <<-SQL
+      result = execute <<~SQL
         SELECT
           table_schema || '.' || table_name AS table_name
         FROM
@@ -578,7 +609,7 @@ module Dexter
 
     def materialized_views
       if server_version_num >= 90300
-        result = execute <<-SQL
+        result = execute <<~SQL
           SELECT
             schemaname || '.' || matviewname AS table_name
           FROM
@@ -595,7 +626,7 @@ module Dexter
     end
 
     def database_view_tables
-      result = execute <<-SQL
+      result = execute <<~SQL
         SELECT
           schemaname || '.' || viewname AS table_name,
           definition
@@ -620,7 +651,8 @@ module Dexter
     end
 
     def stat_statements
-      result = execute <<-SQL
+      total_time = server_version_num >= 130000 ? "(total_plan_time + total_exec_time)" : "total_time"
+      result = execute <<~SQL
         SELECT
           DISTINCT query
         FROM
@@ -629,7 +661,7 @@ module Dexter
           pg_database ON pg_database.oid = pg_stat_statements.dbid
         WHERE
           datname = current_database()
-          AND total_time >= #{@min_time * 60000}
+          AND #{total_time} >= #{@min_time * 60000}
           AND calls >= #{@min_calls}
         ORDER BY
           1
@@ -666,7 +698,7 @@ module Dexter
     end
 
     def columns(tables)
-      columns = execute <<-SQL
+      columns = execute <<~SQL
         SELECT
           s.nspname || '.' || t.relname AS table_name,
           a.attname AS column_name,
@@ -685,7 +717,7 @@ module Dexter
     end
 
     def indexes(tables)
-      execute(<<-SQL
+      query = <<~SQL
         SELECT
           schemaname || '.' || t.relname AS table,
           ix.relname AS name,
@@ -707,7 +739,7 @@ module Dexter
         ORDER BY
           1, 2
       SQL
-      ).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
+      execute(query).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
     end
 
     def search_path

data/lib/dexter/json_log_parser.rb

@@ -1,5 +1,3 @@
-require "json"
-
 module Dexter
   class JsonLogParser < LogParser
     FIRST_LINE_REGEX = /\A.+/

data/lib/dexter/log_parser.rb

@@ -6,35 +6,13 @@ module Dexter
     LINE_SEPERATOR = ":  ".freeze
     DETAIL_LINE = "DETAIL:  ".freeze
 
+    attr_accessor :once
+
     def initialize(logfile, collector)
       @logfile = logfile
       @collector = collector
     end
 
-    def perform
-      active_line = nil
-      duration = nil
-
-      @logfile.each_line do |line|
-        if active_line
-          if line.include?(DETAIL_LINE)
-            add_parameters(active_line, line.chomp.split(DETAIL_LINE)[1])
-          elsif line.include?(LINE_SEPERATOR)
-            process_entry(active_line, duration)
-            active_line = nil
-          else
-            active_line << line
-          end
-        end
-
-        if !active_line && (m = REGEX.match(line.chomp))
-          duration = m[1].to_f
-          active_line = m[3]
-        end
-      end
-      process_entry(active_line, duration) if active_line
-    end
-
     private
 
     def process_entry(query, duration)

data/lib/dexter/pg_stat_activity_parser.rb

@@ -18,6 +18,8 @@ module Dexter
 
         queries = new_queries
 
+        break if once
+
         sleep(1)
       end
     end

data/lib/dexter/processor.rb

@@ -11,6 +11,12 @@ module Dexter
       @log_parser =
         if @logfile == :pg_stat_activity
           PgStatActivityParser.new(@indexer, @collector)
+        elsif @logfile == :log_table
+          if options[:input_format] == "csv"
+            CsvLogTableParser.new(@indexer, @collector)
+          else
+            StderrLogTableParser.new(@indexer, @collector)
+          end
         elsif options[:input_format] == "csv"
           CsvLogParser.new(logfile, @collector)
         elsif options[:input_format] == "json"
@@ -18,11 +24,12 @@ module Dexter
         elsif options[:input_format] == "sql"
           SqlLogParser.new(logfile, @collector)
         else
-          LogParser.new(logfile, @collector)
+          StderrLogParser.new(logfile, @collector)
         end
 
       @starting_interval = 3
       @interval = options[:interval]
+      @log_parser.once = options[:once]
 
       @mutex = Mutex.new
       @last_checked_at = {}
@@ -31,7 +38,7 @@ module Dexter
     end
 
     def perform
-      if [STDIN, :pg_stat_activity].include?(@logfile)
+      if [STDIN, :pg_stat_activity, :log_table].include?(@logfile) && !@log_parser.once
         Thread.abort_on_exception = true
         Thread.new do
           sleep(@starting_interval)

data/lib/dexter/stderr_log_parser.rb

@@ -0,0 +1,31 @@
+module Dexter
+  class StderrLogParser < LogParser
+    def perform
+      process_stderr(@logfile.each_line)
+    end
+
+    def process_stderr(rows)
+      active_line = nil
+      duration = nil
+
+      rows.each do |line|
+        if active_line
+          if line.include?(DETAIL_LINE)
+            add_parameters(active_line, line.chomp.split(DETAIL_LINE)[1])
+          elsif line.include?(LINE_SEPERATOR)
+            process_entry(active_line, duration)
+            active_line = nil
+          else
+            active_line << line
+          end
+        end
+
+        if !active_line && (m = REGEX.match(line.chomp))
+          duration = m[1].to_f
+          active_line = m[3]
+        end
+      end
+      process_entry(active_line, duration) if active_line
+    end
+  end
+end

data/lib/dexter/stderr_log_table_parser.rb

@@ -0,0 +1,7 @@
+module Dexter
+  class StderrLogTableParser < StderrLogParser
+    def perform
+      process_stderr(@logfile.stderr_activity.map { |r| r["log_entry"] })
+    end
+  end
+end

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.4.1"
+  VERSION = "0.4.3"
 end

data/lib/dexter.rb

@@ -4,23 +4,29 @@ require "pg_query"
 require "slop"
 
 # stdlib
+require "csv"
 require "json"
 require "set"
 require "time"
 
 # modules
-require "dexter/version"
-require "dexter/logging"
-require "dexter/client"
-require "dexter/collector"
-require "dexter/indexer"
-require "dexter/log_parser"
-require "dexter/csv_log_parser"
-require "dexter/json_log_parser"
-require "dexter/pg_stat_activity_parser"
-require "dexter/sql_log_parser"
-require "dexter/processor"
-require "dexter/query"
+require_relative "dexter/logging"
+require_relative "dexter/client"
+require_relative "dexter/collector"
+require_relative "dexter/indexer"
+require_relative "dexter/processor"
+require_relative "dexter/query"
+require_relative "dexter/version"
+
+# parsers
+require_relative "dexter/log_parser"
+require_relative "dexter/csv_log_parser"
+require_relative "dexter/csv_log_table_parser"
+require_relative "dexter/json_log_parser"
+require_relative "dexter/pg_stat_activity_parser"
+require_relative "dexter/sql_log_parser"
+require_relative "dexter/stderr_log_parser"
+require_relative "dexter/stderr_log_table_parser"
 
 module Dexter
   class Abort < StandardError; end

metadata

@@ -1,57 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-10-15 00:00:00.000000000 Z
+date: 2023-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: slop
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.8.2
+        version: 0.18.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.8.2
+        version: 0.18.2
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: pg_query
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.18.2
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.18.2
+        version: '2.1'
 - !ruby/object:Gem::Dependency
-  name: pg_query
+  name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 4.10.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 4.10.1
 description:
 email: andrew@ankane.org
 executables:
@@ -67,6 +67,7 @@ files:
 - lib/dexter/client.rb
 - lib/dexter/collector.rb
 - lib/dexter/csv_log_parser.rb
+- lib/dexter/csv_log_table_parser.rb
 - lib/dexter/indexer.rb
 - lib/dexter/json_log_parser.rb
 - lib/dexter/log_parser.rb
@@ -75,6 +76,8 @@ files:
 - lib/dexter/processor.rb
 - lib/dexter/query.rb
 - lib/dexter/sql_log_parser.rb
+- lib/dexter/stderr_log_parser.rb
+- lib/dexter/stderr_log_table_parser.rb
 - lib/dexter/version.rb
 homepage: https://github.com/ankane/dexter
 licenses:
@@ -95,7 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.6
 signing_key:
 specification_version: 4
 summary: The automatic indexer for Postgres