pgdexter 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 14bc122b136301535793c29b19336a2d0614f640
-  data.tar.gz: 624c8c6ae5aabd8a5e2150887aa83efe6e67f9b4
+SHA256:
+  metadata.gz: e51c5c9ce5c31760d773177a3b224674752b19de8cc9fa0978e1da2adff9a490
+  data.tar.gz: ce9b2e89d2524aafda39bd5a91bb53abde5d0819e0c8751281a5ff7ad10766c2
 SHA512:
-  metadata.gz: 44224c687d8590c0441d587b432fde3b85cef77728980059d497ade992cd953e8f8445747de61c984d559f55d74461ac284c6c8423c0f1f8b08e439e361c141d
-  data.tar.gz: 42ec40ec7527dfc3093853ec013c5b30bacbaee5cbe8d27a1530649bfbdc16c639d9d93d65e72b062d53b33bff1d0a1bec67e971d283f5d5a2e6fea7d884fccd
+  metadata.gz: 63cdf589a11ab232f45e26ff8b6a713ef757ce3bc3baf3fecbc51217fb4c3a4bdda0ded52bc631bbb8aaa5fe7ca28d356ab99e200c5af22a3bd1bac9e5efc17b
+  data.tar.gz: fcbb680a98f5907a62216ee142fa91b8538965c1fd59c1de48645ee1b5e3525950245d010a128f52b28fda665d0f6fcbf95fa343184b559150646819049f173a

data/.travis.yml

@@ -6,8 +6,9 @@ addons:
   postgresql: "9.6"
 before_script:
   - sudo apt-get install postgresql-server-dev-9.6
-  - git clone https://github.com/dalibo/hypopg.git
+  - git clone https://github.com/HypoPG/hypopg.git
   - cd hypopg
+  - git checkout 1.1.1
   - make
   - sudo make install
   - psql -c 'create database dexter_test;' -U postgres

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.4
+
+- Fixed `--username` option
+- Fixed `JSON::NestingError`
+- Added `--pg-stat-activity` option
+
 ## 0.3.3
 
 - Added support for views and materialized views

data/README.md

@@ -8,12 +8,12 @@ The automatic indexer for Postgres
 
 ## Installation
 
-First, install [HypoPG](https://github.com/dalibo/hypopg) on your database server. This doesn’t require a restart.
+First, install [HypoPG](https://github.com/HypoPG/hypopg) on your database server. This doesn’t require a restart.
 
 ```sh
 cd /tmp
-curl -L https://github.com/dalibo/hypopg/archive/1.1.0.tar.gz | tar xz
-cd hypopg-1.1.0
+curl -L https://github.com/HypoPG/hypopg/archive/1.1.1.tar.gz | tar xz
+cd hypopg-1.1.1
 make
 make install # may need sudo
 ```
@@ -104,13 +104,19 @@ or pass files:
 dexter <connection-options> <file1> <file2>
 ```
 
+or collect running queries with: [master]
+
+```sh
+dexter <connection-options> --pg-stat-activity
+```
+
 or use the [pg_stat_statements](https://www.postgresql.org/docs/current/static/pgstatstatements.html) extension:
 
 ```sh
 dexter <connection-options> --pg-stat-statements
 ```
 
-> Note: Logs are highly preferred over pg_stat_statements, as pg_stat_statements often doesn’t store enough information to optimize queries.
+> Note: Logs or running queries are highly preferred over pg_stat_statements, as pg_stat_statements often doesn’t store enough information to optimize queries.
 
 ### Collection Options
 

data/guides/Linux.md

@@ -32,6 +32,7 @@ sudo apt-get install dexter
 ### Debian 9 (Stretch)
 
 ```sh
+sudo apt-get -y install apt-transport-https
 wget -qO- https://dl.packager.io/srv/pghero/dexter/key | sudo apt-key add -
 sudo wget -O /etc/apt/sources.list.d/dexter.list \
   https://dl.packager.io/srv/pghero/dexter/master/installer/debian/9.repo
@@ -42,6 +43,7 @@ sudo apt-get install dexter
 ### Debian 8 (Jesse)
 
 ```sh
+sudo apt-get -y install apt-transport-https
 wget -qO- https://dl.packager.io/srv/pghero/dexter/key | sudo apt-key add -
 sudo wget -O /etc/apt/sources.list.d/dexter.list \
   https://dl.packager.io/srv/pghero/dexter/master/installer/debian/8.repo

data/lib/dexter.rb

@@ -11,6 +11,7 @@ require "dexter/collector"
 require "dexter/indexer"
 require "dexter/log_parser"
 require "dexter/csv_log_parser"
+require "dexter/pg_stat_activity_parser"
 require "dexter/processor"
 require "dexter/query"
 

data/lib/dexter/client.rb

@@ -18,6 +18,8 @@ module Dexter
       elsif options[:pg_stat_statements]
         # TODO support streaming option
         Indexer.new(options).process_stat_statements
+      elsif options[:pg_stat_activity]
+        Processor.new(:pg_stat_activity, options).perform
       elsif arguments.any?
         ARGV.replace(arguments)
         Processor.new(ARGF, options).perform
@@ -44,6 +46,7 @@ Options:)
         o.float "--min-calls", "only process queries that have been called a certain number of times", default: 0
         o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
         o.integer "--min-cost-savings-pct", default: 50, help: false
+        o.boolean "--pg-stat-activity", "use pg_stat_activity", default: false, help: false
         o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
         o.string "-s", "--statement", "process a single statement"
         # separator must go here to show up correctly - slop bug?

data/lib/dexter/indexer.rb

@@ -25,6 +25,23 @@ module Dexter
       process_queries(queries)
     end
 
+    def stat_activity
+      execute <<-SQL
+        SELECT
+          pid || ':' || COALESCE(query_start, xact_start) AS id,
+          query,
+          EXTRACT(EPOCH FROM NOW() - COALESCE(query_start, xact_start)) * 1000.0 AS duration_ms
+        FROM
+          pg_stat_activity
+        WHERE
+          datname = current_database()
+          AND state = 'active'
+          AND pid != pg_backend_pid()
+        ORDER BY
+          1
+      SQL
+    end
+
     def process_queries(queries)
       # reset hypothetical indexes
       reset_hypothetical_indexes
@@ -166,7 +183,7 @@ module Dexter
             # Pass format to prevent ANALYZE
             puts execute("EXPLAIN (FORMAT TEXT) #{safe_statement(query.statement)}").map { |r| r["QUERY PLAN"] }.join("\n")
           end
-        rescue PG::Error => e
+        rescue PG::Error, JSON::NestingError => e
           if @log_explain
             log e.message
           end
@@ -479,6 +496,9 @@ module Dexter
 
     def conn
       @conn ||= begin
+        # set connect timeout if none set
+        ENV["PGCONNECT_TIMEOUT"] ||= "2"
+
         if @options[:dbname] =~ /\Apostgres(ql)?:\/\//
           config = @options[:dbname]
         else
@@ -486,7 +506,7 @@ module Dexter
             host: @options[:host],
             port: @options[:port],
             dbname: @options[:dbname],
-            user: @options[:user]
+            user: @options[:username]
           }.reject { |_, value| value.to_s.empty? }
           config = config[:dbname] if config.keys == [:dbname] && config[:dbname].include?("=")
         end
@@ -511,7 +531,7 @@ module Dexter
 
     def plan(query)
       # strip semi-colons as another measure of defense
-      JSON.parse(execute("EXPLAIN (FORMAT JSON) #{safe_statement(query)}").first["QUERY PLAN"]).first["Plan"]
+      JSON.parse(execute("EXPLAIN (FORMAT JSON) #{safe_statement(query)}").first["QUERY PLAN"], max_nesting: 1000).first["Plan"]
     end
 
     # TODO for multicolumn indexes, use ordering

data/lib/dexter/pg_stat_activity_parser.rb

@@ -0,0 +1,25 @@
+module Dexter
+  class PgStatActivityParser < LogParser
+    def perform
+      queries = {}
+
+      loop do
+        new_queries = {}
+        @logfile.stat_activity.each do |row|
+          new_queries[row["id"]] = row
+        end
+
+        # store queries after they complete
+        queries.each do |id, row|
+          unless new_queries[id]
+            process_entry(row["query"], row["duration_ms"].to_f)
+          end
+        end
+
+        queries = new_queries
+
+        sleep(5)
+      end
+    end
+  end
+end

data/lib/dexter/processor.rb

@@ -6,15 +6,17 @@ module Dexter
       @logfile = logfile
 
       @collector = Collector.new(min_time: options[:min_time], min_calls: options[:min_calls])
+      @indexer = Indexer.new(options)
+
       @log_parser =
-        if options[:input_format] == "csv"
+        if @logfile == :pg_stat_activity
+          PgStatActivityParser.new(@indexer, @collector)
+        elsif options[:input_format] == "csv"
           CsvLogParser.new(logfile, @collector)
         else
           LogParser.new(logfile, @collector)
         end
 
-      @indexer = Indexer.new(options)
-
       @starting_interval = 3
       @interval = options[:interval]
 
@@ -25,7 +27,7 @@ module Dexter
     end
 
     def perform
-      if @logfile == STDIN
+      if [STDIN, :pg_stat_activity].include?(@logfile)
         Thread.abort_on_exception = true
         Thread.new do
           sleep(@starting_interval)

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.3.3"
+  VERSION = "0.3.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-02-23 00:00:00.000000000 Z
+date: 2018-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -119,6 +119,7 @@ files:
 - lib/dexter/indexer.rb
 - lib/dexter/log_parser.rb
 - lib/dexter/logging.rb
+- lib/dexter/pg_stat_activity_parser.rb
 - lib/dexter/processor.rb
 - lib/dexter/query.rb
 - lib/dexter/version.rb
@@ -142,7 +143,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: The automatic indexer for Postgres