pgdexter 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a9dd13cbe8dc39e26b90bdc13abd16201ccec455
-  data.tar.gz: bc4693ab0595c9dc1e38b720c294b8e600d597c5
+  metadata.gz: b472c8289c8c878c0ea790a3ec5fdf43e22bfaa1
+  data.tar.gz: 55f96eb8bd0a8739c79fb7c749a8aee68b2a119f
 SHA512:
-  metadata.gz: 5f96e60ac1660786dd0f059ec0e88a4f017c056f73eb0be07bb6d0d9ad55332a3c36c72cd9fb7f8b3a5ed370254b361f29979baa031850d93800ce8ad74775d2
-  data.tar.gz: f6b5a5a8487ea44eb38a4aca3ec59aaaea6c8cdd2c3e40267203fa29b4999d03a507fbe8bbe7e4207a5110c7bcc1e649dba624b4eb11e859be1f432e705095a8
+  metadata.gz: 873d3d0f9471f90c9b90721274623c6c3dcfbcfe88989143a6b66a5ce1345a2187bab75ef4f32f52efd9f34a1e1806524696854348e592d6eb31bec25ca193c8
+  data.tar.gz: a545b2b1a9d2f6650ec04c1769b243863c412fbff3b1e0575ab4cc6c98648119e487a2bc75f0afeeb704895184580ada13169a05edb3e00bc0caca37759183bc

data/CHANGELOG.md

@@ -0,0 +1,14 @@
+## 0.1.2
+
+- Added `--exclude` option
+- Added `--log-sql` option
+
+## 0.1.1
+
+- Added `--interval` option
+- Added `--min-time` option
+- Added `--log-level` option
+
+## 0.1.0
+
+- Launched

data/README.md

@@ -1,6 +1,8 @@
 # Dexter
 
-An automatic indexer for Postgres
+The automatic indexer for Postgres
+
+[Read about how it works](https://medium.com/@ankane/introducing-dexter-the-automatic-indexer-for-postgres-5f8fa8b28f27)
 
 ## Installation
 
@@ -22,7 +24,7 @@ Enable logging for slow queries.
 log_min_duration_statement = 10 # ms
 ```
 
-And install with:
+And install the command line tool with:
 
 ```sh
 gem install pgdexter
@@ -50,12 +52,31 @@ This finds slow queries and generates output like:
 2017-06-25T17:53:22+00:00 Processing 12 new query fingerprints
 ```
 
-To be safe, Dexter will not create indexes unless you pass the `--create` flag.
+To be safe, Dexter will not create indexes unless you pass the `--create` flag. In this case, you’ll see:
+
+```log
+2017-06-25T17:52:22+00:00 Index found: ratings (user_id)
+2017-06-25T17:52:22+00:00 Creating index: CREATE INDEX CONCURRENTLY ON ratings (user_id)
+2017-06-25T17:52:37+00:00 Index created: 15243 ms
+```
 
 ## Options
 
-- `--interval` - time to wait between processing queries
-- `--min-time` - only consider queries that have consumed a certain amount of DB time (in minutes)
+Name | Description | Default
+--- | --- | ---
+exclude | prevent specific tables from being indexed | None
+interval | time to wait between processing queries, in seconds | 60
+log-level | `debug` gives additional info for suggested indexes<br />`debug2` gives additional info for all processed queries | info
+log-sql | log SQL statements executed | false
+min-time | only process queries consuming a min amount of DB time, in minutes | 0
+
+## Future Work
+
+[Here are some ideas](https://github.com/ankane/dexter/issues/1)
+
+## Thanks
+
+This software wouldn’t be possible without [HypoPG](https://github.com/dalibo/hypopg), which allows you to create hypothetical indexes, and [pg_query](https://github.com/lfittl/pg_query), which allows you to parse and fingerprint queries. A big thanks to Dalibo and Lukas Fittl respectively.
 
 ## Contributing
 

data/lib/dexter.rb

@@ -5,49 +5,10 @@ require "pg_query"
 require "time"
 require "set"
 require "thread"
+require "dexter/logging"
+require "dexter/client"
+require "dexter/collector"
 require "dexter/indexer"
 require "dexter/log_parser"
-
-module Dexter
-  class Client
-    attr_reader :arguments, :options
-
-    def initialize(args)
-      @arguments, @options = parse_args(args)
-    end
-
-    def perform
-      abort "Missing database url" if arguments.empty?
-      abort "Too many arguments" if arguments.size > 2
-
-      # get queries
-      queries = []
-      if options[:s]
-        queries << options[:s]
-        Indexer.new(self).process_queries(queries)
-      end
-      if arguments[1]
-        begin
-          LogParser.new(arguments[1], self).perform
-        rescue Errno::ENOENT
-          abort "Log file not found"
-        end
-      end
-      if !options[:s] && !arguments[1]
-        LogParser.new(STDIN, self).perform
-      end
-    end
-
-    def parse_args(args)
-      opts = Slop.parse(args) do |o|
-        o.boolean "--create", default: false
-        o.string "-s"
-        o.float "--min-time", default: 0
-        o.integer "--interval", default: 60
-      end
-      [opts.arguments, opts.to_hash]
-    rescue Slop::Error => e
-      abort e.message
-    end
-  end
-end
+require "dexter/processor"
+require "dexter/query"

data/lib/dexter/client.rb

@@ -0,0 +1,67 @@
+module Dexter
+  class Client
+    attr_reader :arguments, :options
+
+    def initialize(args)
+      @arguments, @options = parse_args(args)
+    end
+
+    def perform
+      STDOUT.sync = true
+      STDERR.sync = true
+
+      if options[:statement]
+        fingerprint = PgQuery.fingerprint(options[:statement]) rescue "unknown"
+        query = Query.new(options[:statement], fingerprint)
+        Indexer.new(arguments[0], options).process_queries([query])
+      elsif arguments[1]
+        Processor.new(arguments[0], arguments[1], options).perform
+      else
+        Processor.new(arguments[0], STDIN, options).perform
+      end
+    end
+
+    def parse_args(args)
+      opts = Slop.parse(args) do |o|
+        o.banner = %(Usage:
+    dexter <database-url> [options]
+
+Options:)
+        o.boolean "--create", "create indexes", default: false
+        o.array "--exclude", "prevent specific tables from being indexed"
+        o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
+        o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
+        o.string "--log-level", "log level", default: "info"
+        o.boolean "--log-sql", "log sql", default: false
+        o.string "-s", "--statement", "process a single statement"
+        o.on "-v", "--version", "print the version" do
+          log Dexter::VERSION
+          exit
+        end
+        o.on "-h", "--help", "prints help" do
+          log o
+          exit
+        end
+      end
+
+      arguments = opts.arguments
+
+      if arguments.empty?
+        log opts
+        exit
+      end
+
+      abort "Too many arguments" if arguments.size > 2
+
+      abort "Unknown log level" unless ["info", "debug", "debug2"].include?(opts.to_hash[:log_level].to_s.downcase)
+
+      [arguments, opts.to_hash]
+    rescue Slop::Error => e
+      abort e.message
+    end
+
+    def log(message)
+      $stderr.puts message
+    end
+  end
+end

data/lib/dexter/collector.rb

@@ -0,0 +1,47 @@
+module Dexter
+  class Collector
+    def initialize(options = {})
+      @top_queries = {}
+      @new_queries = Set.new
+      @mutex = Mutex.new
+      @min_time = options[:min_time] * 60000 # convert minutes to ms
+    end
+
+    def add(query, duration)
+      fingerprint =
+        begin
+          PgQuery.fingerprint(query)
+        rescue PgQuery::ParseError
+          # do nothing
+        end
+
+      return unless fingerprint
+
+      @top_queries[fingerprint] ||= {calls: 0, total_time: 0}
+      @top_queries[fingerprint][:calls] += 1
+      @top_queries[fingerprint][:total_time] += duration
+      @top_queries[fingerprint][:query] = query
+      @mutex.synchronize do
+        @new_queries << fingerprint
+      end
+    end
+
+    def fetch_queries
+      new_queries = nil
+
+      @mutex.synchronize do
+        new_queries = @new_queries.dup
+        @new_queries.clear
+      end
+
+      queries = []
+      @top_queries.each do |k, v|
+        if new_queries.include?(k) && v[:total_time] > @min_time
+          queries << Query.new(v[:query], k)
+        end
+      end
+
+      queries.sort_by(&:fingerprint)
+    end
+  end
+end

data/lib/dexter/indexer.rb

@@ -1,36 +1,100 @@
 module Dexter
   class Indexer
-    attr_reader :client
+    include Logging
 
-    def initialize(client)
-      @client = client
+    def initialize(database_url, options)
+      @database_url = database_url
+      @create = options[:create]
+      @log_level = options[:log_level]
+      @exclude_tables = options[:exclude]
+      @log_sql = options[:log_sql]
 
-      select_all("SET client_min_messages = warning")
-      select_all("CREATE EXTENSION IF NOT EXISTS hypopg")
+      create_extension
     end
 
     def process_queries(queries)
-      # narrow down queries and tables
-      tables, queries = narrow_queries(queries)
-      return [] if tables.empty?
+      # reset hypothetical indexes
+      reset_hypothetical_indexes
+
+      # filter queries from other databases and system tables
+      tables = possible_tables(queries)
+      queries.each do |query|
+        query.missing_tables = !query.tables.all? { |t| tables.include?(t) }
+      end
+
+      # exclude user specified tables
+      # TODO exclude write-heavy tables
+      @exclude_tables.each do |table|
+        tables.delete(table)
+      end
+
+      # analyze tables if needed
+      analyze_tables(tables) if tables.any?
+
+      # get initial costs for queries
+      calculate_initial_cost(queries.reject(&:missing_tables))
+
+      # create hypothetical indexes
+      candidates = tables.any? ? create_hypothetical_indexes(tables) : {}
+
+      # get new costs and see if new indexes were used
+      new_indexes = determine_indexes(queries, candidates)
+
+      # display and create new indexes
+      show_and_create_indexes(new_indexes)
+    end
+
+    private
 
-      # get ready for hypothetical indexes
+    def create_extension
+      select_all("SET client_min_messages = warning")
+      select_all("CREATE EXTENSION IF NOT EXISTS hypopg")
+    end
+
+    def reset_hypothetical_indexes
       select_all("SELECT hypopg_reset()")
+    end
 
-      # ensure tables have recently been analyzed
-      analyze_tables(tables)
+    def analyze_tables(tables)
+      tables = tables.to_a.sort
+
+      analyze_stats = select_all <<-SQL
+        SELECT
+          schemaname AS schema,
+          relname AS table,
+          last_analyze,
+          last_autoanalyze
+        FROM
+          pg_stat_user_tables
+        WHERE
+          relname IN (#{tables.map { |t| quote(t) }.join(", ")})
+      SQL
+
+      last_analyzed = {}
+      analyze_stats.each do |stats|
+        last_analyzed[stats["table"]] = Time.parse(stats["last_analyze"]) if stats["last_analyze"]
+      end
 
-      # get initial plans
-      initial_plans = {}
+      tables.each do |table|
+        if !last_analyzed[table] || last_analyzed[table] < Time.now - 3600
+          statement = "ANALYZE #{quote_ident(table)}"
+          log "Running analyze: #{statement}"
+          select_all(statement)
+        end
+      end
+    end
+
+    def calculate_initial_cost(queries)
       queries.each do |query|
         begin
-          initial_plans[query] = plan(query)
+          query.initial_cost = plan(query.statement)["Total Cost"]
         rescue PG::Error
           # do nothing
         end
       end
-      queries.select! { |q| initial_plans[q] }
+    end
 
+    def create_hypothetical_indexes(tables)
       # get existing indexes
       index_set = Set.new
       indexes(tables).each do |index|
@@ -45,60 +109,96 @@ module Dexter
           candidates[col] = select_all("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{col[:table]} (#{[col[:column]].join(", ")})');").first["indexname"]
         end
       end
+      candidates
+    end
 
-      queries_by_index = {}
+    def determine_indexes(queries, candidates)
+      new_indexes = {}
 
-      new_indexes = []
       queries.each do |query|
-        starting_cost = initial_plans[query]["Total Cost"]
-        plan2 = plan(query)
-        cost2 = plan2["Total Cost"]
-        best_indexes = []
-
-        candidates.each do |col, index_name|
-          if plan2.inspect.include?(index_name) && cost2 < starting_cost * 0.5
-            best_indexes << {
-              table: col[:table],
-              columns: [col[:column]]
-            }
-            (queries_by_index[best_indexes.last] ||= []) << {
-              starting_cost: starting_cost,
-              final_cost: cost2,
-              query: query
-            }
+        if query.initial_cost
+          new_plan = plan(query.statement)
+          query.new_cost = new_plan["Total Cost"]
+          cost_savings = query.new_cost < query.initial_cost * 0.5
+
+          query_indexes = []
+          candidates.each do |col, index_name|
+            if new_plan.inspect.include?(index_name)
+              index = {
+                table: col[:table],
+                columns: [col[:column]]
+              }
+              query_indexes << index
+
+              if cost_savings
+                new_indexes[index] ||= index.dup
+                (new_indexes[index][:queries] ||= []) << query
+              end
+            end
           end
         end
 
-        new_indexes.concat(best_indexes)
+        if @log_level == "debug2"
+          log "Processed #{query.fingerprint}"
+          if query.initial_cost
+            log "Cost: #{query.initial_cost} -> #{query.new_cost}"
+
+            if query_indexes.any?
+              log "Indexes: #{query_indexes.map { |i| "#{i[:table]} (#{i[:columns].join(", ")})" }.join(", ")}"
+              log "Need 50% cost savings to suggest index" unless cost_savings
+            else
+              log "Indexes: None"
+            end
+          elsif query.fingerprint == "unknown"
+            log "Could not parse query"
+          elsif query.tables.empty?
+            log "No tables"
+          elsif query.missing_tables
+            log "Tables not present in current database"
+          else
+            log "Could not run explain"
+          end
+
+          puts
+          puts query.statement
+          puts
+        end
       end
 
-      new_indexes = new_indexes.uniq.sort_by(&:to_a)
+      new_indexes.values.sort_by(&:to_a)
+    end
 
-      # create indexes
+    def show_and_create_indexes(new_indexes)
       if new_indexes.any?
         new_indexes.each do |index|
-          index[:queries] = queries_by_index[index]
-
           log "Index found: #{index[:table]} (#{index[:columns].join(", ")})"
-          # log "CREATE INDEX CONCURRENTLY ON #{index[:table]} (#{index[:columns].join(", ")});"
-          # index[:queries].sort_by { |q| fingerprints[q[:query]] }.each do |query|
-          #   log "Query #{fingerprints[query[:query]]} (Cost: #{query[:starting_cost]} -> #{query[:final_cost]})"
-          #   puts
-          #   puts query[:query]
-          #   puts
-          # end
+
+          if @log_level.start_with?("debug")
+            index[:queries].sort_by(&:fingerprint).each do |query|
+              log "Query #{query.fingerprint} (Cost: #{query.initial_cost} -> #{query.new_cost})"
+              puts
+              puts query.statement
+              puts
+            end
+          end
         end
 
-        new_indexes.each do |index|
-          statement = "CREATE INDEX CONCURRENTLY ON #{index[:table]} (#{index[:columns].join(", ")})"
-          # puts "#{statement};"
-          if client.options[:create]
+        if @create
+          # TODO use advisory locks
+          # 1. create lock
+          # 2. refresh existing index list
+          # 3. create indexes that still don't exist
+          # 4. release lock
+          new_indexes.each do |index|
+            statement = "CREATE INDEX CONCURRENTLY ON #{index[:table]} (#{index[:columns].join(", ")})"
             log "Creating index: #{statement}"
             started_at = Time.now
             select_all(statement)
             log "Index created: #{((Time.now - started_at) * 1000).to_i} ms"
           end
         end
+      else
+        log "No indexes found"
       end
 
       new_indexes
@@ -106,7 +206,7 @@ module Dexter
 
     def conn
       @conn ||= begin
-        uri = URI.parse(client.arguments[0])
+        uri = URI.parse(@database_url)
         config = {
           host: uri.host,
           port: uri.port,
@@ -122,14 +222,24 @@ module Dexter
     end
 
     def select_all(query)
-      conn.exec(query).to_a
+      # use exec_params instead of exec for securiy
+      #
+      # Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
+      # (There can be semicolons in it, but not more than one nonempty command.)
+      # This is a limitation of the underlying protocol, but has some usefulness
+      # as an extra defense against SQL-injection attacks.
+      # https://www.postgresql.org/docs/current/static/libpq-exec.html
+      query = squish(query)
+      log "SQL: #{query}" if @log_sql
+      conn.exec_params(query, []).to_a
     end
 
     def plan(query)
-      JSON.parse(select_all("EXPLAIN (FORMAT JSON) #{query}").first["QUERY PLAN"]).first["Plan"]
+      # strip semi-colons as another measure of defense
+      JSON.parse(select_all("EXPLAIN (FORMAT JSON) #{query.gsub(";", "")}").first["QUERY PLAN"]).first["Plan"]
     end
 
-    def narrow_queries(queries)
+    def database_tables
       result = select_all <<-SQL
         SELECT
           table_name
@@ -139,11 +249,11 @@ module Dexter
           table_catalog = current_database() AND
           table_schema NOT IN ('pg_catalog', 'information_schema')
       SQL
-      possible_tables = Set.new(result.map { |r| r["table_name"] })
-
-      tables = queries.flat_map { |q| PgQuery.parse(q).tables }.uniq.select { |t| possible_tables.include?(t) }
+      result.map { |r| r["table_name"] }
+    end
 
-      [tables, queries.select { |q| PgQuery.parse(q).tables.all? { |t| possible_tables.include?(t) } }]
+    def possible_tables(queries)
+      Set.new(queries.flat_map(&:tables).uniq & database_tables)
     end
 
     def columns(tables)
@@ -168,13 +278,7 @@ module Dexter
           t.relname AS table,
           ix.relname AS name,
           regexp_replace(pg_get_indexdef(i.indexrelid), '^[^\\(]*\\((.*)\\)$', '\\1') AS columns,
-          regexp_replace(pg_get_indexdef(i.indexrelid), '.* USING ([^ ]*) \\(.*', '\\1') AS using,
-          indisunique AS unique,
-          indisprimary AS primary,
-          indisvalid AS valid,
-          indexprs::text,
-          indpred::text,
-          pg_get_indexdef(i.indexrelid) AS definition
+          regexp_replace(pg_get_indexdef(i.indexrelid), '.* USING ([^ ]*) \\(.*', '\\1') AS using
         FROM
           pg_index i
         INNER JOIN
@@ -203,30 +307,8 @@ module Dexter
       end
     end
 
-    def analyze_tables(tables)
-      analyze_stats = select_all <<-SQL
-        SELECT
-          schemaname AS schema,
-          relname AS table,
-          last_analyze,
-          last_autoanalyze
-        FROM
-          pg_stat_user_tables
-        WHERE
-          relname IN (#{tables.map { |t| quote(t) }.join(", ")})
-      SQL
-
-      last_analyzed = {}
-      analyze_stats.each do |stats|
-        last_analyzed[stats["table"]] = Time.parse(stats["last_analyze"]) if stats["last_analyze"]
-      end
-
-      tables.each do |table|
-        if !last_analyzed[table] || last_analyzed[table] < Time.now - 3600
-          log "Analyzing #{table}"
-          select_all("ANALYZE #{table}")
-        end
-      end
+    def quote_ident(value)
+      conn.quote_ident(value)
     end
 
     def quote(value)
@@ -237,13 +319,14 @@ module Dexter
       end
     end
 
-    # activerecord
+    # from activerecord
     def quote_string(s)
       s.gsub(/\\/, '\&\&').gsub(/'/, "''")
     end
 
-    def log(message)
-      puts "#{Time.now.iso8601} #{message}"
+    # from activesupport
+    def squish(str)
+      str.to_s.gsub(/\A[[:space:]]+/, "").gsub(/[[:space:]]+\z/, "").gsub(/[[:space:]]+/, " ")
     end
   end
 end

data/lib/dexter/log_parser.rb

@@ -1,32 +1,11 @@
 module Dexter
   class LogParser
     REGEX = /duration: (\d+\.\d+) ms  (statement|execute <unnamed>): (.+)/
+    LINE_SEPERATOR = ":  ".freeze
 
-    def initialize(logfile, client)
+    def initialize(logfile, collector)
       @logfile = logfile
-      @min_time = client.options[:min_time] * 60000 # convert minutes to ms
-      @top_queries = {}
-      @indexer = Indexer.new(client)
-      @new_queries = Set.new
-      @new_queries_mutex = Mutex.new
-      @process_queries_mutex = Mutex.new
-      @last_checked_at = {}
-
-      log "Started"
-
-      if @logfile == STDIN
-        Thread.abort_on_exception = true
-
-        @timer_thread = Thread.new do
-          sleep(3) # starting sleep
-          loop do
-            @process_queries_mutex.synchronize do
-              process_queries
-            end
-            sleep(client.options[:interval])
-          end
-        end
-      end
+      @collector = collector
     end
 
     def perform
@@ -35,10 +14,9 @@ module Dexter
 
       each_line do |line|
         if active_line
-          if line.include?(":  ")
+          if line.include?(LINE_SEPERATOR)
             process_entry(active_line, duration)
             active_line = nil
-            duration = nil
           else
             active_line << line
           end
@@ -47,15 +25,9 @@ module Dexter
         if !active_line && m = REGEX.match(line.chomp)
           duration = m[1].to_f
           active_line = m[3]
-        else
-          # skip
         end
       end
       process_entry(active_line, duration) if active_line
-
-      @process_queries_mutex.synchronize do
-        process_queries
-      end
     end
 
     private
@@ -66,59 +38,18 @@ module Dexter
           yield line
         end
       else
-        File.foreach(@logfile) do |line|
-          yield line
-        end
-      end
-    end
-
-    def process_entry(query, duration)
-      return unless query =~ /SELECT/i
-      fingerprint =
         begin
-          PgQuery.fingerprint(query)
-        rescue PgQuery::ParseError
-          # do nothing
-        end
-      return unless fingerprint
-
-      @top_queries[fingerprint] ||= {calls: 0, total_time: 0}
-      @top_queries[fingerprint][:calls] += 1
-      @top_queries[fingerprint][:total_time] += duration
-      @top_queries[fingerprint][:query] = query
-      @new_queries_mutex.synchronize do
-        @new_queries << fingerprint
-      end
-    end
-
-    def process_queries
-      new_queries = nil
-
-      @new_queries_mutex.synchronize do
-        new_queries = @new_queries.dup
-        @new_queries.clear
-      end
-
-      now = Time.now
-      min_checked_at = now - 3600 # don't recheck for an hour
-      queries = []
-      fingerprints = {}
-      @top_queries.each do |k, v|
-        if new_queries.include?(k) && v[:total_time] > @min_time && (!@last_checked_at[k] || @last_checked_at[k] < min_checked_at)
-          fingerprints[v[:query]] = k
-          queries << v[:query]
-          @last_checked_at[k] = now
+          File.foreach(@logfile) do |line|
+            yield line
+          end
+        rescue Errno::ENOENT
+          abort "Log file not found"
         end
       end
-
-      log "Processing #{queries.size} new query fingerprints"
-      if queries.any?
-        @indexer.process_queries(queries)
-      end
     end
 
-    def log(message)
-      puts "#{Time.now.iso8601} #{message}"
+    def process_entry(query, duration)
+      @collector.add(query, duration) if query =~ /SELECT/i
     end
   end
 end

data/lib/dexter/logging.rb

@@ -0,0 +1,7 @@
+module Dexter
+  module Logging
+    def log(message)
+      puts "#{Time.now.iso8601} #{message}"
+    end
+  end
+end

data/lib/dexter/processor.rb

@@ -0,0 +1,61 @@
+module Dexter
+  class Processor
+    include Logging
+
+    def initialize(database_url, logfile, options)
+      log "Started"
+
+      @logfile = logfile
+
+      @collector = Collector.new(min_time: options[:min_time])
+      @log_parser = LogParser.new(logfile, @collector)
+      @indexer = Indexer.new(database_url, options)
+
+      @starting_interval = 3
+      @interval = options[:interval]
+
+      @mutex = Mutex.new
+      @last_checked_at = {}
+    end
+
+    def perform
+      if @logfile == STDIN
+        Thread.abort_on_exception = true
+        Thread.new do
+          sleep(@starting_interval)
+          loop do
+            process_queries
+            sleep(@interval)
+          end
+        end
+      end
+
+      @log_parser.perform
+
+      process_queries
+    end
+
+    private
+
+    def process_queries
+      @mutex.synchronize do
+        process_queries_without_lock
+      end
+    end
+
+    def process_queries_without_lock
+      now = Time.now
+      min_checked_at = now - 3600 # don't recheck for an hour
+      queries = []
+      @collector.fetch_queries.each do |query|
+        if !@last_checked_at[query.fingerprint] || @last_checked_at[query.fingerprint] < min_checked_at
+          queries << query
+          @last_checked_at[query.fingerprint] = now
+        end
+      end
+
+      log "Processing #{queries.size} new query fingerprints"
+      @indexer.process_queries(queries) if queries.any?
+    end
+  end
+end

data/lib/dexter/query.rb

@@ -0,0 +1,15 @@
+module Dexter
+  class Query
+    attr_reader :statement, :fingerprint
+    attr_accessor :initial_cost, :new_cost, :missing_tables
+
+    def initialize(statement, fingerprint)
+      @statement = statement
+      @fingerprint = fingerprint
+    end
+
+    def tables
+      @tables ||= PgQuery.parse(statement).tables rescue []
+    end
+  end
+end

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/pgdexter.gemspec

@@ -1,4 +1,5 @@
 # coding: utf-8
+
 lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "dexter/version"
@@ -9,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Andrew Kane"]
   spec.email         = ["andrew@chartkick.com"]
 
-  spec.summary       = "An automatic indexer for Postgres"
+  spec.summary       = "The automatic indexer for Postgres"
   spec.homepage      = "https://github.com/ankane/dexter"
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-25 00:00:00.000000000 Z
+date: 2017-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -89,14 +89,20 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - exe/dexter
 - lib/dexter.rb
+- lib/dexter/client.rb
+- lib/dexter/collector.rb
 - lib/dexter/indexer.rb
 - lib/dexter/log_parser.rb
+- lib/dexter/logging.rb
+- lib/dexter/processor.rb
+- lib/dexter/query.rb
 - lib/dexter/version.rb
 - pgdexter.gemspec
 homepage: https://github.com/ankane/dexter
@@ -121,5 +127,5 @@ rubyforge_project:
 rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
-summary: An automatic indexer for Postgres
+summary: The automatic indexer for Postgres
 test_files: []