pgdexter 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 16f61626f5003a801ec12cda3f663390a57e69d8
-  data.tar.gz: 238ee1b24dab3f473accc2ca9a5156fd31e8d19e
+  metadata.gz: 78c0747c456e498798accd4e4881a8041d246e07
+  data.tar.gz: c4c9cffe3e04d07675c36abf0532db3634e356ea
 SHA512:
-  metadata.gz: ccae6266196fc84ea0fc69177b80144f8747996ff0693b6a7492e34d095b2cb400e9ce3299741326c50a5eb267ef48be368384a2ab27553002eb93ac7b6d0922
-  data.tar.gz: 944bc27bb1ffff546c2f33aa10aa1b8371ba43ae2be889a6b9445c3d3a93ca36fee0cfd57487134a5ea807e37583f74b1e5e9de7714fa30364ab28f64a5679e8
+  metadata.gz: df826f95f34809bd94cb7450cc43d4bd6beed81606e6e7079842dd4a05e983d4041bf956645c09a290cf8fa37140917da1a12fe332d6268969da5310f688e148
+  data.tar.gz: f7015b408e7864bfe7e4e136cbb60acbe837cbbe640ec4f43dfb2d9631f2d4cb2b4c86826cb58051d68941bdfa6fd6625029babe7c1609e084db1a4f92a23652

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.1
+
+- Added support for queries with bind variables
+- Fixed error with streaming logs as csv format
+- Handle malformed CSV gracefully
+
 ## 0.3.0
 
 - Added support for schemas

data/README.md

@@ -130,7 +130,7 @@ Specify the format
 dexter --input-format csv
 ```
 
-When steaming logs, specify the time to wait between processing queries
+When streaming logs, specify the time to wait between processing queries
 
 ```sh
 dexter --interval 60 # seconds
@@ -150,6 +150,14 @@ Homebrew on Mac
 tail -F -n +1 /usr/local/var/postgres/server.log | dexter dbname
 ```
 
+## Analyze
+
+For best results, make sure your tables have been recently analyzed so statistics are up-to-date. You can ask Dexter to analyze tables it comes across that haven’t been analyzed in the past hour with:
+
+```sh
+dexter --analyze
+```
+
 ## Tables
 
 You can exclude large or write-heavy tables from indexing with:
@@ -172,14 +180,6 @@ See how Dexter is processing queries with:
 dexter --log-sql --log-level debug2
 ```
 
-## Analyze
-
-For best results, make sure your tables have been recently analyzed so statistics are up-to-date. You can ask Dexter to analyze tables it comes across that haven’t been analyzed in the past hour with:
-
-```sh
-dexter --analyze
-```
-
 ## Hosted Postgres
 
 Some hosted providers like Amazon RDS and Heroku do not support the HypoPG extension, which Dexter needs to run. See [how to use Dexter](guides/Hosted-Postgres.md) in these cases.

data/lib/dexter/csv_log_parser.rb

@@ -3,11 +3,15 @@ require "csv"
 module Dexter
   class CsvLogParser < LogParser
     def perform
-      CSV.foreach(@logfile.file) do |row|
+      CSV.new(@logfile).each do |row|
         if (m = REGEX.match(row[13]))
-          process_entry(m[3], m[1].to_f)
+          active_line = m[3]
+          add_parameters(active_line, row[14]) if row[14]
+          process_entry(active_line, m[1].to_f)
         end
       end
+    rescue CSV::MalformedCSVError => e
+      abort "ERROR: #{e.message}"
     end
   end
 end

data/lib/dexter/indexer.rb

@@ -130,18 +130,22 @@ module Dexter
 
     def calculate_plan(queries)
       queries.each do |query|
+        if @log_explain
+          puts "Explaining query"
+          puts
+        end
         begin
           query.plans << plan(query.statement)
           if @log_explain
-            log "Explaining query"
-            puts
             # Pass format to prevent ANALYZE
             puts execute("EXPLAIN (FORMAT TEXT) #{safe_statement(query.statement)}").map { |r| r["QUERY PLAN"] }.join("\n")
-            puts
           end
-        rescue PG::Error
-          # do nothing
+        rescue PG::Error => e
+          if @log_explain
+            log e.message
+          end
         end
+        puts if @log_explain
       end
     end
 

data/lib/dexter/log_parser.rb

@@ -1,7 +1,8 @@
 module Dexter
   class LogParser
-    REGEX = /duration: (\d+\.\d+) ms  (statement|execute <unnamed>|parse <unnamed>): (.+)/
+    REGEX = /duration: (\d+\.\d+) ms  (statement|execute <unnamed>): (.+)/
     LINE_SEPERATOR = ":  ".freeze
+    DETAIL_LINE = "DETAIL:  ".freeze
 
     def initialize(logfile, collector)
       @logfile = logfile
@@ -14,7 +15,9 @@ module Dexter
 
       @logfile.each_line do |line|
         if active_line
-          if line.include?(LINE_SEPERATOR)
+          if line.include?(DETAIL_LINE)
+            add_parameters(active_line, line.chomp.split(DETAIL_LINE)[1])
+          elsif line.include?(LINE_SEPERATOR)
             process_entry(active_line, duration)
             active_line = nil
           else
@@ -35,5 +38,18 @@ module Dexter
     def process_entry(query, duration)
       @collector.add(query, duration)
     end
+
+    def add_parameters(active_line, details)
+      if details.start_with?("parameters: ")
+        params = Hash[details[12..-1].split(", ").map { |s| s.split(" = ", 2) }]
+
+        # make sure parsing was successful
+        unless params.values.include?(nil)
+          params.each do |k, v|
+            active_line.sub!(k, v)
+          end
+        end
+      end
+    end
   end
 end

data/lib/dexter/processor.rb

@@ -43,7 +43,7 @@ module Dexter
       begin
         @log_parser.perform
       rescue Errno::ENOENT => e
-        abort e.message
+        abort "ERROR: #{e.message}"
       end
 
       process_queries

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-12-23 00:00:00.000000000 Z
+date: 2017-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop