pgdexter 0.5.6 → 0.6.0

data/lib/dexter/parsers/csv_log_parser.rb

@@ -0,0 +1,25 @@
+module Dexter
+  class CsvLogParser < LogParser
+    FIRST_LINE_REGEX = /\A.+/
+
+    def perform(collector)
+      CSV.new(@logfile.to_io).each do |row|
+        message = row[13]
+        detail = row[14]
+
+        if (m = REGEX.match(message))
+          # replace first line with match
+          # needed for multiline queries
+          active_line = message.sub(FIRST_LINE_REGEX, m[3])
+
+          add_parameters(active_line, detail) if detail
+          collector.add(active_line, m[1].to_f)
+        end
+      end
+    rescue CSV::MalformedCSVError => e
+      raise Error, "ERROR: #{e.message}"
+    ensure
+      @logfile.close
+    end
+  end
+end

data/lib/dexter/{json_log_parser.rb → parsers/json_log_parser.rb}

@@ -2,7 +2,7 @@ module Dexter
   class JsonLogParser < LogParser
     FIRST_LINE_REGEX = /\A.+/
 
-    def perform
+    def perform(collector)
       @logfile.each_line do |line|
         row = JSON.parse(line.chomp)
         if (m = REGEX.match(row["message"]))
@@ -11,11 +11,13 @@ module Dexter
           active_line = row["message"].sub(FIRST_LINE_REGEX, m[3])
 
           add_parameters(active_line, row["detail"]) if row["detail"]
-          process_entry(active_line, m[1].to_f)
+          collector.add(active_line, m[1].to_f)
         end
       end
     rescue JSON::ParserError => e
-      raise Dexter::Abort, "ERROR: #{e.message}"
+      raise Error, "ERROR: #{e.message}"
+    ensure
+      @logfile.close
     end
   end
 end

data/lib/dexter/{log_parser.rb → parsers/log_parser.rb}

@@ -1,20 +1,13 @@
 module Dexter
   class LogParser
-    include Logging
-
     REGEX = /duration: (\d+\.\d+) ms  (statement|execute [^:]+): (.+)/
 
-    def initialize(logfile, collector)
+    def initialize(logfile)
       @logfile = logfile
-      @collector = collector
     end
 
     private
 
-    def process_entry(query, duration)
-      @collector.add(query, duration)
-    end
-
     def add_parameters(active_line, details)
       if details.start_with?("parameters: ")
         params = Hash[details[12..-1].split(", ").map { |s| s.split(" = ", 2) }]

data/lib/dexter/{sql_log_parser.rb → parsers/sql_log_parser.rb}

@@ -1,9 +1,10 @@
 module Dexter
   class SqlLogParser < LogParser
-    def perform
+    def perform(collector)
       # TODO support streaming
       @logfile.read.split(";").each do |statement|
-        process_entry(statement, 0)
+        statement = statement.strip
+        collector.add(statement, 0) unless statement.empty?
       end
     end
   end

data/lib/dexter/{stderr_log_parser.rb → parsers/stderr_log_parser.rb}

@@ -3,20 +3,16 @@ module Dexter
     LINE_SEPERATOR = ":  ".freeze
     DETAIL_LINE = "DETAIL:  ".freeze
 
-    def perform
-      process_stderr(@logfile.each_line)
-    end
-
-    def process_stderr(rows)
+    def perform(collector)
       active_line = nil
       duration = nil
 
-      rows.each do |line|
+      @logfile.each_line do |line|
         if active_line
           if line.include?(DETAIL_LINE)
             add_parameters(active_line, line.chomp.split(DETAIL_LINE)[1])
           elsif line.include?(LINE_SEPERATOR)
-            process_entry(active_line, duration)
+            collector.add(active_line, duration)
             active_line = nil
           else
             active_line << line
@@ -28,7 +24,7 @@ module Dexter
           active_line = m[3]
         end
       end
-      process_entry(active_line, duration) if active_line
+      collector.add(active_line, duration) if active_line
     end
   end
 end

data/lib/dexter/processor.rb

@@ -2,36 +2,22 @@ module Dexter
   class Processor
     include Logging
 
-    def initialize(logfile, options)
-      @logfile = logfile
-
-      @collector = Collector.new(min_time: options[:min_time], min_calls: options[:min_calls])
-      @indexer = Indexer.new(options)
-
-      @log_parser =
-        if @logfile == :pg_stat_activity
-          PgStatActivityParser.new(@indexer, @collector)
-        elsif options[:input_format] == "csv"
-          CsvLogParser.new(logfile, @collector)
-        elsif options[:input_format] == "json"
-          JsonLogParser.new(logfile, @collector)
-        elsif options[:input_format] == "sql"
-          SqlLogParser.new(logfile, @collector)
-        else
-          StderrLogParser.new(logfile, @collector)
-        end
+    def initialize(source, collector, indexer, interval:)
+      @source = source
+      @collector = collector
+      @indexer = indexer
 
       @starting_interval = 3
-      @interval = options[:interval]
+      @interval = interval
 
       @mutex = Mutex.new
       @last_checked_at = {}
 
-      log "Started"
+      log "Started" if !@source.is_a?(PgStatStatementsSource) && !@source.is_a?(StatementSource)
     end
 
     def perform
-      if [STDIN].include?(@logfile)
+      if @source.is_a?(LogSource) && @source.stdin?
         Thread.abort_on_exception = true
         Thread.new do
           sleep(@starting_interval)
@@ -47,9 +33,9 @@ module Dexter
       end
 
       begin
-        @log_parser.perform
+        @source.perform(@collector)
       rescue Errno::ENOENT => e
-        raise Dexter::Abort, "ERROR: #{e.message}"
+        raise Error, "ERROR: #{e.message}"
       end
 
       process_queries
@@ -74,7 +60,7 @@ module Dexter
         end
       end
 
-      log "Processing #{queries.size} new query fingerprints"
+      log "Processing #{queries.size} new query fingerprints" unless @source.is_a?(StatementSource)
       @indexer.process_queries(queries) if queries.any?
     end
   end

data/lib/dexter/query.rb

@@ -1,36 +1,32 @@
 module Dexter
   class Query
-    attr_reader :statement, :fingerprint, :plans
-    attr_writer :tables
-    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes, :pass3_indexes, :candidate_tables, :tables_from_views, :candidates
+    attr_reader :statement, :fingerprint, :total_time, :calls, :plans
+    attr_accessor :tables, :missing_tables, :new_cost, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes, :pass3_indexes, :candidate_tables, :tables_from_views, :index_mapping, :columns, :candidate_columns
 
-    def initialize(statement, fingerprint = nil)
+    def initialize(statement, fingerprint = nil, total_time: nil, calls: nil)
       @statement = statement
-      unless fingerprint
-        fingerprint = PgQuery.fingerprint(statement) rescue "unknown"
-      end
       @fingerprint = fingerprint
+      @total_time = total_time
+      @calls = calls
       @plans = []
       @tables_from_views = []
+      @candidate_tables = []
+      @columns = []
+      @candidate_columns = []
     end
 
-    def tables
-      @tables ||= begin
-        parse ? parse.tables : []
-      rescue => e
-        # possible pg_query bug
-        $stderr.puts "Error extracting tables. Please report to https://github.com/ankane/dexter/issues"
-        $stderr.puts "#{e.class.name}: #{e.message}"
-        $stderr.puts statement
-        []
+    def parser_result
+      unless defined?(@parser_result)
+        @parser_result = PgQuery.parse(statement) rescue nil
       end
+      @parser_result
     end
 
     def tree
-      parse.tree
+      parser_result.tree
     end
 
-    def explainable?
+    def fully_analyzed?
       plans.size >= 3
     end
 
@@ -41,18 +37,5 @@ module Dexter
     def initial_cost
       costs[0]
     end
-
-    def high_cost?
-      initial_cost && initial_cost >= 100
-    end
-
-    private
-
-    def parse
-      unless defined?(@parse)
-        @parse = PgQuery.parse(statement) rescue nil
-      end
-      @parse
-    end
   end
 end

data/lib/dexter/sources/log_source.rb

@@ -0,0 +1,26 @@
+module Dexter
+  class LogSource
+    def initialize(logfile, input_format)
+      @log_parser =
+        case input_format
+        when "csv"
+          CsvLogParser.new(logfile)
+        when "json"
+          JsonLogParser.new(logfile)
+        when "sql"
+          SqlLogParser.new(logfile)
+        else
+          StderrLogParser.new(logfile)
+        end
+      @stdin = logfile == STDIN
+    end
+
+    def perform(collector)
+      @log_parser.perform(collector)
+    end
+
+    def stdin?
+      @stdin
+    end
+  end
+end

data/lib/dexter/{pg_stat_activity_parser.rb → sources/pg_stat_activity_source.rb}

@@ -1,6 +1,10 @@
 module Dexter
-  class PgStatActivityParser < LogParser
-    def perform
+  class PgStatActivitySource
+    def initialize(connection)
+      @connection = connection
+    end
+
+    def perform(collector)
       previous_queries = {}
 
       10.times do
@@ -11,7 +15,7 @@ module Dexter
           if row["state"] == "active"
             active_queries[row["id"]] = row
           else
-            process_entry(row["query"], row["duration_ms"].to_f)
+            collector.add(row["query"], row["duration_ms"].to_f)
             processed_queries[row["id"]] = true
           end
         end
@@ -19,13 +23,13 @@ module Dexter
         # store queries after they complete
         previous_queries.each do |id, row|
           if !active_queries[id] && !processed_queries[id]
-            process_entry(row["query"], row["duration_ms"].to_f)
+            collector.add(row["query"], row["duration_ms"].to_f)
           end
         end
 
         previous_queries = active_queries
 
-        sleep(0.1)
+        sleep($dexter_test ? 0 : 0.1)
       end
     end
 
@@ -44,7 +48,7 @@ module Dexter
         ORDER BY
           1
       SQL
-      @logfile.send(:execute, sql)
+      @connection.execute(sql)
     end
   end
 end

data/lib/dexter/sources/pg_stat_statements_source.rb

@@ -0,0 +1,34 @@
+module Dexter
+  class PgStatStatementsSource
+    def initialize(connection)
+      @connection = connection
+    end
+
+    def perform(collector)
+      stat_statements.each do |row|
+        collector.add(row["query"], row["duration_ms"].to_f, row["calls"].to_i)
+      end
+    end
+
+    # could group, sum, and filter min_time/min_calls in SQL, but keep simple for now
+    def stat_statements
+      sql = <<~SQL
+        SELECT
+          query,
+          total_plan_time + total_exec_time AS duration_ms,
+          calls
+        FROM
+          pg_stat_statements
+        INNER JOIN
+          pg_database ON pg_database.oid = pg_stat_statements.dbid
+        WHERE
+          datname = current_database()
+        ORDER BY
+          1
+      SQL
+      @connection.execute(sql)
+    rescue PG::UndefinedTable => e
+      raise Error, e.message
+    end
+  end
+end

data/lib/dexter/sources/statement_source.rb

@@ -0,0 +1,11 @@
+module Dexter
+  class StatementSource
+    def initialize(statement)
+      @statement = statement
+    end
+
+    def perform(collector)
+      collector.add(@statement, 0, 0, true)
+    end
+  end
+end

data/lib/dexter/table_resolver.rb

@@ -0,0 +1,120 @@
+module Dexter
+  class TableResolver
+    include Logging
+
+    def initialize(connection, queries, log_level:)
+      @connection = connection
+      @queries = queries
+      @log_level = log_level
+    end
+
+    def perform
+      tables = Set.new(database_tables + materialized_views)
+      no_schema_tables = self.no_schema_tables(tables)
+      view_tables = self.view_tables(no_schema_tables)
+
+      @queries.each do |query|
+        # add schema to table if needed
+        query_tables = self.tables(query).map { |t| no_schema_tables[t] || t }
+
+        # substitute view tables
+        query.tables = query_tables.flat_map { |t| view_tables[t] || [t] }.uniq
+        query.tables_from_views = query.tables - query_tables
+
+        query.missing_tables = !query.tables.all? { |t| tables.include?(t) }
+      end
+    end
+
+    private
+
+    def tables(query)
+      query.parser_result&.tables || []
+    rescue => e
+      # possible pg_query bug
+      $stderr.puts "Error extracting tables. Please report to https://github.com/ankane/dexter/issues"
+      $stderr.puts "#{e.class.name}: #{e.message}"
+      $stderr.puts query.statement
+      []
+    end
+
+    def no_schema_tables(tables)
+      search_path_index = Hash[search_path.map.with_index.to_a]
+      tables.group_by { |t| t.split(".")[-1] }.to_h do |group, t2|
+        [group, t2.sort_by { |t| [search_path_index[t.split(".")[0]] || 1000000, t] }[0]]
+      end
+    end
+
+    def view_tables(no_schema_tables)
+      # add tables from views
+      view_tables = database_view_tables
+      view_tables.each do |v, vt|
+        view_tables[v] = vt.map { |t| no_schema_tables[t] || t }
+      end
+
+      # fully resolve tables
+      # make sure no views in result
+      view_tables.each do |v, vt|
+        view_tables[v] = vt.flat_map { |t| view_tables[t] || [t] }.uniq
+      end
+
+      view_tables
+    end
+
+    def execute(...)
+      @connection.execute(...)
+    end
+
+    def search_path
+      execute("SELECT current_schemas(true)")[0]["current_schemas"][1..-2].split(",")
+    end
+
+    def database_tables
+      result = execute <<~SQL
+        SELECT
+          table_schema || '.' || table_name AS table_name
+        FROM
+          information_schema.tables
+        WHERE
+          table_catalog = current_database()
+          AND table_type IN ('BASE TABLE', 'VIEW')
+      SQL
+      result.map { |r| r["table_name"] }
+    end
+
+    def materialized_views
+      result = execute <<~SQL
+        SELECT
+          schemaname || '.' || matviewname AS table_name
+        FROM
+          pg_matviews
+      SQL
+      result.map { |r| r["table_name"] }
+    end
+
+    def views
+      execute <<~SQL
+        SELECT
+          schemaname || '.' || viewname AS table_name,
+          definition
+        FROM
+          pg_views
+        WHERE
+          schemaname NOT IN ('information_schema', 'pg_catalog')
+      SQL
+    end
+
+    def database_view_tables
+      view_tables = {}
+      views.each do |row|
+        begin
+          view_tables[row["table_name"]] = PgQuery.parse(row["definition"]).tables
+        rescue PgQuery::ParseError
+          if @log_level.start_with?("debug")
+            log colorize("ERROR: Cannot parse view definition: #{row["table_name"]}", :red)
+          end
+        end
+      end
+      view_tables
+    end
+  end
+end

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.5.6"
+  VERSION = "0.6.0"
 end

data/lib/dexter.rb

@@ -13,19 +13,27 @@ require "time"
 require_relative "dexter/logging"
 require_relative "dexter/client"
 require_relative "dexter/collector"
+require_relative "dexter/column_resolver"
+require_relative "dexter/connection"
 require_relative "dexter/indexer"
 require_relative "dexter/processor"
 require_relative "dexter/query"
+require_relative "dexter/table_resolver"
 require_relative "dexter/version"
 
 # parsers
-require_relative "dexter/log_parser"
-require_relative "dexter/csv_log_parser"
-require_relative "dexter/json_log_parser"
-require_relative "dexter/pg_stat_activity_parser"
-require_relative "dexter/sql_log_parser"
-require_relative "dexter/stderr_log_parser"
+require_relative "dexter/parsers/log_parser"
+require_relative "dexter/parsers/csv_log_parser"
+require_relative "dexter/parsers/json_log_parser"
+require_relative "dexter/parsers/sql_log_parser"
+require_relative "dexter/parsers/stderr_log_parser"
+
+# sources
+require_relative "dexter/sources/log_source"
+require_relative "dexter/sources/pg_stat_activity_source"
+require_relative "dexter/sources/pg_stat_statements_source"
+require_relative "dexter/sources/statement_source"
 
 module Dexter
-  class Abort < StandardError; end
+  class Error < StandardError; end
 end

data/lib/pgdexter.rb

@@ -0,0 +1 @@
+require_relative "dexter"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.6.0
 platform: ruby
 authors:
 - Andrew Kane
 bindir: exe
 cert_chain: []
-date: 2025-02-01 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -29,14 +29,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.18.2
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.18.2
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: pg_query
   requirement: !ruby/object:Gem::Requirement
@@ -78,17 +78,24 @@ files:
 - lib/dexter.rb
 - lib/dexter/client.rb
 - lib/dexter/collector.rb
-- lib/dexter/csv_log_parser.rb
+- lib/dexter/column_resolver.rb
+- lib/dexter/connection.rb
 - lib/dexter/indexer.rb
-- lib/dexter/json_log_parser.rb
-- lib/dexter/log_parser.rb
 - lib/dexter/logging.rb
-- lib/dexter/pg_stat_activity_parser.rb
+- lib/dexter/parsers/csv_log_parser.rb
+- lib/dexter/parsers/json_log_parser.rb
+- lib/dexter/parsers/log_parser.rb
+- lib/dexter/parsers/sql_log_parser.rb
+- lib/dexter/parsers/stderr_log_parser.rb
 - lib/dexter/processor.rb
 - lib/dexter/query.rb
-- lib/dexter/sql_log_parser.rb
-- lib/dexter/stderr_log_parser.rb
+- lib/dexter/sources/log_source.rb
+- lib/dexter/sources/pg_stat_activity_source.rb
+- lib/dexter/sources/pg_stat_statements_source.rb
+- lib/dexter/sources/statement_source.rb
+- lib/dexter/table_resolver.rb
 - lib/dexter/version.rb
+- lib/pgdexter.rb
 homepage: https://github.com/ankane/dexter
 licenses:
 - MIT
@@ -100,14 +107,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.7
 specification_version: 4
 summary: The automatic indexer for Postgres
 test_files: []

data/lib/dexter/csv_log_parser.rb

@@ -1,24 +0,0 @@
-module Dexter
-  class CsvLogParser < LogParser
-    FIRST_LINE_REGEX = /\A.+/
-
-    def perform
-      CSV.new(@logfile.to_io).each do |row|
-        process_csv_row(row[13], row[14])
-      end
-    rescue CSV::MalformedCSVError => e
-      raise Dexter::Abort, "ERROR: #{e.message}"
-    end
-
-    def process_csv_row(message, detail)
-      if (m = REGEX.match(message))
-        # replace first line with match
-        # needed for multiline queries
-        active_line = message.sub(FIRST_LINE_REGEX, m[3])
-
-        add_parameters(active_line, detail) if detail
-        process_entry(active_line, m[1].to_f)
-      end
-    end
-  end
-end