pgdexter 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d08be58d7de049df2c11f33211c0ada165f4a363
-  data.tar.gz: ce17e302b12f3b7babe38de9618b76bab982b9d9
+  metadata.gz: fbc105f249fc27a10fcd47fa16ba9d732f0cd933
+  data.tar.gz: 20929a8fd1d268194b9bc2c7bab71130a2eabc43
 SHA512:
-  metadata.gz: 7e0974f0f8c39ec8039749a75a3fb61f7613c6931db15f191eba2dda2a794cf57e8875f6aff4465474a9c608727deef08fd7a5833e16ba96e2d1d5a02e3e7f7b
-  data.tar.gz: 73668e8c819b4334888bec707db491453ab2cc03a39e189e451738bb7606b8fe329912e2520d6e2b6b7aeb9634a8e2200b267887257c26428bf453fde74cc54e
+  metadata.gz: b01d8e32db41bdf7fa0dd2412b4b490318d8574fbfba5ed0357f1aa9f3e8dcc2e9a0fbef9ee80376678c184f8840a5bf5e52fef5b325cdbffb2b31106d5c2838
+  data.tar.gz: 2e9808dba0e161eb184ddc180ee915f0c49097225b2539a11916eaf5dd32e9bedce8ef07763391f758d6f4627258c90e6af4f7c158b473323d398050e64a9f75

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.1.5
+
+- Added support for non-`SELECT` queries
+- Added `--pg-stat-statements` option
+- Added advisory locks
+- Added support for running as a non-superuser
+
 ## 0.1.4
 
 - Added support for multicolumn indexes

data/README.md

@@ -37,7 +37,7 @@ The command line tool is also available as a [Linux package](guides/Linux.md).
 Dexter needs a connection to your database and a log file to process.
 
 ```sh
-tail -F -n +1 <log-file> | dexter <database-url>
+tail -F -n +1 <log-file> | dexter <connection-string>
 ```
 
 This finds slow queries and generates output like:
@@ -62,12 +62,18 @@ To be safe, Dexter will not create indexes unless you pass the `--create` flag.
 2017-06-25T17:52:37+00:00 Index created: 15243 ms
 ```
 
-## Single Statement Mode
+## Connection String
 
-You can also pass a single statement with:
+The connection string is a URI with the format:
 
-```sh
-dexter <database-url> -s "SELECT * FROM ..."
+```
+postgresql://user:pass@host:5432/dbname
+```
+
+To connect through a socket, just pass the database name.
+
+```
+dbname
 ```
 
 ## Options
@@ -80,6 +86,28 @@ log-level | `debug` gives additional info for suggested indexes<br />`debug2` gi
 log-sql | log SQL statements executed | false
 min-time | only process queries consuming a min amount of DB time, in minutes | 0
 
+## Single Statement Mode
+
+You can pass a single statement with:
+
+```sh
+dexter <connection-string> -s "SELECT * FROM ..."
+```
+
+## Examples
+
+Ubuntu with PostgreSQL 9.6
+
+```sh
+tail -F -n +1 /var/log/postgresql/postgresql-9.6-main.log | sudo -u postgres dexter dbname
+```
+
+Homebrew on Mac
+
+```sh
+tail -F -n +1 /usr/local/var/postgres/server.log | dexter dbname
+```
+
 ## Future Work
 
 [Here are some ideas](https://github.com/ankane/dexter/issues/1)

data/guides/Linux.md

@@ -11,8 +11,9 @@ Distributions
 ### Ubuntu 16.04 (Xenial)
 
 ```sh
-wget -qO - https://deb.packager.io/key | sudo apt-key add -
-echo "deb https://deb.packager.io/gh/pghero/dexter xenial master" | sudo tee /etc/apt/sources.list.d/dexter.list
+wget -qO- https://dl.packager.io/srv/pghero/dexter/key | sudo apt-key add -
+sudo wget -O /etc/apt/sources.list.d/dexter.list \
+  https://dl.packager.io/srv/pghero/dexter/master/installer/ubuntu/16.04.repo
 sudo apt-get update
 sudo apt-get -y install dexter
 ```
@@ -20,8 +21,9 @@ sudo apt-get -y install dexter
 ### Ubuntu 14.04 (Trusty)
 
 ```sh
-wget -qO - https://deb.packager.io/key | sudo apt-key add -
-echo "deb https://deb.packager.io/gh/pghero/dexter trusty master" | sudo tee /etc/apt/sources.list.d/dexter.list
+wget -qO- https://dl.packager.io/srv/pghero/dexter/key | sudo apt-key add -
+sudo wget -O /etc/apt/sources.list.d/dexter.list \
+  https://dl.packager.io/srv/pghero/dexter/master/installer/ubuntu/14.04.repo
 sudo apt-get update
 sudo apt-get install dexter
 ```
@@ -29,8 +31,9 @@ sudo apt-get install dexter
 ### Debian 8 (Jesse)
 
 ```sh
-wget -qO - https://deb.packager.io/key | sudo apt-key add -
-echo "deb https://deb.packager.io/gh/pghero/dexter jessie master" | sudo tee /etc/apt/sources.list.d/dexter.list
+wget -qO- https://dl.packager.io/srv/pghero/dexter/key | sudo apt-key add -
+sudo wget -O /etc/apt/sources.list.d/dexter.list \
+  https://dl.packager.io/srv/pghero/dexter/master/installer/debian/8.repo
 sudo apt-get update
 sudo apt-get install dexter
 ```
@@ -38,19 +41,16 @@ sudo apt-get install dexter
 ### CentOS / RHEL 7
 
 ```sh
-sudo rpm --import https://rpm.packager.io/key
-echo "[dexter]
-name=Repository for pghero/dexter application.
-baseurl=https://rpm.packager.io/gh/pghero/dexter/centos7/master
-enabled=1" | sudo tee /etc/yum.repos.d/dexter.repo
+sudo wget -O /etc/yum.repos.d/dexter.repo \
+  https://dl.packager.io/srv/pghero/dexter/master/installer/el/7.repo
 sudo yum install dexter
 ```
 
 ### SUSE Linux Enterprise Server 12
 
 ```sh
-sudo rpm --import https://rpm.packager.io/key
-sudo zypper addrepo "https://rpm.packager.io/gh/pghero/dexter/sles12/master" "dexter"
+sudo wget -O /etc/zypp/repos.d/dexter.repo \
+  https://dl.packager.io/srv/pghero/dexter/master/installer/sles/12.repo
 sudo zypper install dexter
 ```
 

data/lib/dexter/client.rb

@@ -11,9 +11,10 @@ module Dexter
       STDERR.sync = true
 
       if options[:statement]
-        fingerprint = PgQuery.fingerprint(options[:statement]) rescue "unknown"
-        query = Query.new(options[:statement], fingerprint)
+        query = Query.new(options[:statement])
         Indexer.new(arguments[0], options).process_queries([query])
+      elsif options[:pg_stat_statements]
+        Indexer.new(arguments[0], options).process_stat_statements
       elsif arguments[1]
         Processor.new(arguments[0], arguments[1], options).perform
       else
@@ -31,6 +32,7 @@ Options:)
         o.array "--exclude", "prevent specific tables from being indexed"
         o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
         o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
+        o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
         o.boolean "--log-explain", "log explain", default: false, help: false
         o.string "--log-level", "log level", default: "info"
         o.boolean "--log-sql", "log sql", default: false

data/lib/dexter/indexer.rb

@@ -9,8 +9,15 @@ module Dexter
       @exclude_tables = options[:exclude]
       @log_sql = options[:log_sql]
       @log_explain = options[:log_explain]
+      @min_time = options[:min_time] || 0
 
-      create_extension
+      create_extension unless extension_exists?
+    end
+
+    def process_stat_statements
+      queries = stat_statements.map { |q| Query.new(q) }.sort_by(&:fingerprint).group_by(&:fingerprint).map { |_, v| v.first }
+      log "Processing #{queries.size} new query fingerprints"
+      process_queries(queries)
     end
 
     def process_queries(queries)
@@ -46,7 +53,15 @@ module Dexter
 
     def create_extension
       execute("SET client_min_messages = warning")
-      execute("CREATE EXTENSION IF NOT EXISTS hypopg")
+      begin
+        execute("CREATE EXTENSION IF NOT EXISTS hypopg")
+      rescue PG::InsufficientPrivilege
+        abort "Use a superuser to run: CREATE EXTENSION hypopg"
+      end
+    end
+
+    def extension_exists?
+      execute("SELECT * FROM pg_available_extensions WHERE name = 'hypopg' AND installed_version IS NOT NULL").any?
     end
 
     def reset_hypothetical_indexes
@@ -208,17 +223,20 @@ module Dexter
         end
 
         if @create
-          # TODO use advisory locks
           # 1. create lock
           # 2. refresh existing index list
           # 3. create indexes that still don't exist
           # 4. release lock
-          new_indexes.each do |index|
-            statement = "CREATE INDEX CONCURRENTLY ON #{quote_ident(index[:table])} (#{index[:columns].map { |c| quote_ident(c) }.join(", ")})"
-            log "Creating index: #{statement}"
-            started_at = Time.now
-            execute(statement)
-            log "Index created: #{((Time.now - started_at) * 1000).to_i} ms"
+          with_advisory_lock do
+            new_indexes.each do |index|
+              unless index_exists?(index)
+                statement = "CREATE INDEX CONCURRENTLY ON #{quote_ident(index[:table])} (#{index[:columns].map { |c| quote_ident(c) }.join(", ")})"
+                log "Creating index: #{statement}"
+                started_at = Time.now
+                execute(statement)
+                log "Index created: #{((Time.now - started_at) * 1000).to_i} ms"
+              end
+            end
           end
         end
       else
@@ -242,7 +260,7 @@ module Dexter
         PG::Connection.new(config)
       end
     rescue PG::ConnectionBad
-      abort "Bad database url"
+      abort "Can't connect to database"
     end
 
     def execute(query)
@@ -284,14 +302,60 @@ module Dexter
         WHERE
           table_catalog = current_database() AND
           table_schema NOT IN ('pg_catalog', 'information_schema')
+          AND table_type = 'BASE TABLE'
       SQL
       result.map { |r| r["table_name"] }
     end
 
+    def stat_statements
+      result = execute <<-SQL
+        SELECT
+          DISTINCT query
+        FROM
+          pg_stat_statements
+        INNER JOIN
+          pg_database ON pg_database.oid = pg_stat_statements.dbid
+        WHERE
+          datname = current_database()
+          AND total_time >= #{@min_time * 60000}
+        ORDER BY
+          1
+      SQL
+      result.map { |q| q["query"] }
+    end
+
     def possible_tables(queries)
       Set.new(queries.flat_map(&:tables).uniq & database_tables)
     end
 
+    def with_advisory_lock
+      lock_id = 123456
+      first_time = true
+      while execute("SELECT pg_try_advisory_lock(#{lock_id})").first["pg_try_advisory_lock"] != "t"
+        if first_time
+          log "Waiting for lock..."
+          first_time = false
+        end
+        sleep(1)
+      end
+      yield
+    ensure
+      with_min_messages("error") do
+        execute("SELECT pg_advisory_unlock(#{lock_id})")
+      end
+    end
+
+    def with_min_messages(value)
+      execute("SET client_min_messages = #{quote(value)}")
+      yield
+    ensure
+      execute("SET client_min_messages = warning")
+    end
+
+    def index_exists?(index)
+      indexes([index[:table]]).find { |i| i["columns"] == index[:columns] }
+    end
+
     def columns(tables)
       columns = execute <<-SQL
         SELECT

data/lib/dexter/log_parser.rb

@@ -49,7 +49,7 @@ module Dexter
     end
 
     def process_entry(query, duration)
-      @collector.add(query, duration) if query =~ /SELECT/i
+      @collector.add(query, duration)
     end
   end
 end

data/lib/dexter/query.rb

@@ -3,8 +3,11 @@ module Dexter
     attr_reader :statement, :fingerprint, :plans
     attr_accessor :missing_tables, :new_cost
 
-    def initialize(statement, fingerprint)
+    def initialize(statement, fingerprint = nil)
       @statement = statement
+      unless fingerprint
+        fingerprint = PgQuery.fingerprint(statement) rescue "unknown"
+      end
       @fingerprint = fingerprint
       @plans = []
     end

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-03 00:00:00.000000000 Z
+date: 2017-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop