pgdexter 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e51c5c9ce5c31760d773177a3b224674752b19de8cc9fa0978e1da2adff9a490
-  data.tar.gz: ce9b2e89d2524aafda39bd5a91bb53abde5d0819e0c8751281a5ff7ad10766c2
+  metadata.gz: c797cdeb10797636a6406b017fc9276ea6522656f6592769b5088bee56165cfb
+  data.tar.gz: 76dc688fcea9ccde4aead259ef582f6103d62a5f40e7be668ee2b03347a14470
 SHA512:
-  metadata.gz: 63cdf589a11ab232f45e26ff8b6a713ef757ce3bc3baf3fecbc51217fb4c3a4bdda0ded52bc631bbb8aaa5fe7ca28d356ab99e200c5af22a3bd1bac9e5efc17b
-  data.tar.gz: fcbb680a98f5907a62216ee142fa91b8538965c1fd59c1de48645ee1b5e3525950245d010a128f52b28fda665d0f6fcbf95fa343184b559150646819049f173a
+  metadata.gz: a5ac6bf812b56648ac0b1f864baf025b7cd4e81cccf6d6de65f1780fd63f8f1e640aff53aca771cece0ce54267ad7a428aaba8f42773591380547fac05bfe94a
+  data.tar.gz: 7c0f8b36467cc5eacbc96f6e5e2132de3526ea9827cab9c5ede7e1eff2b471f22948e0b202f1a00fd8b37b785934eec8508ad561947e50efa9b9ffb07e5eb9ae

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.5
+
+- Added `sql` input format
+- Fixed error for queries with double dash comments
+- Fixed connection threading issue with `--pg-stat-activity` option
+
 ## 0.3.4
 
 - Fixed `--username` option

data/README.md

@@ -104,7 +104,7 @@ or pass files:
 dexter <connection-options> <file1> <file2>
 ```
 
-or collect running queries with: [master]
+or collect running queries with:
 
 ```sh
 dexter <connection-options> --pg-stat-activity
@@ -215,6 +215,10 @@ gem specific_install https://github.com/ankane/dexter.git
 
 This software wouldn’t be possible without [HypoPG](https://github.com/dalibo/hypopg), which allows you to create hypothetical indexes, and [pg_query](https://github.com/lfittl/pg_query), which allows you to parse and fingerprint queries. A big thanks to Dalibo and Lukas Fittl respectively.
 
+## Research
+
+This is known as the Index Selection Problem (ISP).
+
 ## Contributing
 
 Everyone is encouraged to help improve this project. Here are a few ways you can help:
@@ -236,5 +240,6 @@ rake install
 To run tests, use:
 
 ```sh
+createdb dexter_test
 rake test
 ```

data/lib/dexter.rb

@@ -12,6 +12,7 @@ require "dexter/indexer"
 require "dexter/log_parser"
 require "dexter/csv_log_parser"
 require "dexter/pg_stat_activity_parser"
+require "dexter/sql_log_parser"
 require "dexter/processor"
 require "dexter/query"
 

data/lib/dexter/indexer.rb

@@ -14,6 +14,7 @@ module Dexter
       @analyze = options[:analyze]
       @min_cost_savings_pct = options[:min_cost_savings_pct].to_i
       @options = options
+      @mutex = Mutex.new
 
       create_extension unless extension_exists?
       execute("SET lock_timeout = '5s'")
@@ -181,7 +182,7 @@ module Dexter
           query.plans << plan(query.statement)
           if @log_explain
             # Pass format to prevent ANALYZE
-            puts execute("EXPLAIN (FORMAT TEXT) #{safe_statement(query.statement)}").map { |r| r["QUERY PLAN"] }.join("\n")
+            puts execute("EXPLAIN (FORMAT TEXT) #{safe_statement(query.statement)}", pretty: false).map { |r| r["QUERY PLAN"] }.join("\n")
           end
         rescue PG::Error, JSON::NestingError => e
           if @log_explain
@@ -362,11 +363,12 @@ module Dexter
                 winning_cost < query.initial_cost * savings_ratio
               end
 
+            query_indexes = [winning_index]
+            new_cost3 = winning_cost
+            query.pass3_indexes = query_indexes
+
             if use_winning
-              query_indexes = [winning_index]
               cost_savings3 = true
-              new_cost3 = winning_cost
-              query.pass3_indexes = query_indexes
             else
               suggest_index = false
             end
@@ -456,7 +458,7 @@ module Dexter
               log "Pass3: #{query.costs[3]} : #{log_indexes(query.pass3_indexes || [])}"
             end
             log "Final: #{query.new_cost} : #{log_indexes(query.suggest_index ? query_indexes : [])}"
-            if query_indexes.size == 1 && !query.suggest_index
+            if (query.pass1_indexes.any? || query.pass2_indexes.any?) && !query.suggest_index
               log "Need #{@min_cost_savings_pct}% cost savings to suggest index"
             end
           else
@@ -516,7 +518,7 @@ module Dexter
       abort e.message
     end
 
-    def execute(query)
+    def execute(query, pretty: true)
       # use exec_params instead of exec for security
       #
       # Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
@@ -524,14 +526,17 @@ module Dexter
       # This is a limitation of the underlying protocol, but has some usefulness
       # as an extra defense against SQL-injection attacks.
       # https://www.postgresql.org/docs/current/static/libpq-exec.html
-      query = squish(query)
+      query = squish(query) if pretty
       log "SQL: #{query}" if @log_sql
-      conn.exec_params(query, []).to_a
+
+      @mutex.synchronize do
+        conn.exec_params(query, []).to_a
+      end
     end
 
     def plan(query)
       # strip semi-colons as another measure of defense
-      JSON.parse(execute("EXPLAIN (FORMAT JSON) #{safe_statement(query)}").first["QUERY PLAN"], max_nesting: 1000).first["Plan"]
+      JSON.parse(execute("EXPLAIN (FORMAT JSON) #{safe_statement(query)}", pretty: false).first["QUERY PLAN"], max_nesting: 1000).first["Plan"]
     end
 
     # TODO for multicolumn indexes, use ordering

data/lib/dexter/pg_stat_activity_parser.rb

@@ -18,7 +18,7 @@ module Dexter
 
         queries = new_queries
 
-        sleep(5)
+        sleep(1)
       end
     end
   end

data/lib/dexter/processor.rb

@@ -13,6 +13,8 @@ module Dexter
           PgStatActivityParser.new(@indexer, @collector)
         elsif options[:input_format] == "csv"
           CsvLogParser.new(logfile, @collector)
+        elsif options[:input_format] == "sql"
+          SqlLogParser.new(logfile, @collector)
         else
           LogParser.new(logfile, @collector)
         end

data/lib/dexter/sql_log_parser.rb

@@ -0,0 +1,10 @@
+module Dexter
+  class SqlLogParser < LogParser
+    def perform
+      # TODO support streaming
+      @logfile.read.split(";").each do |statement|
+        process_entry(statement, 1)
+      end
+    end
+  end
+end

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.3.4"
+  VERSION = "0.3.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-10 00:00:00.000000000 Z
+date: 2018-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -122,6 +122,7 @@ files:
 - lib/dexter/pg_stat_activity_parser.rb
 - lib/dexter/processor.rb
 - lib/dexter/query.rb
+- lib/dexter/sql_log_parser.rb
 - lib/dexter/version.rb
 - pgdexter.gemspec
 homepage: https://github.com/ankane/dexter