pgdexter 0.3.10 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c604783bd2ae16f3bb648176f96664b31fb6ac50a0094e3d34df195e0a4bb9ee
-  data.tar.gz: 299487f0880aa486c2678063f1f6bed4fc2398603b5fd9f2ebeb1c27b31cfb26
+  metadata.gz: 109808872c6a579e6ce82956d5b2b1da8c66af7c85fae3d5984367e374484eb2
+  data.tar.gz: 6fe0f25f316afd84c287ba2fefe5076846c8a572bf2cbee406843abf632ddb2d
 SHA512:
-  metadata.gz: 3cbec0f51a8b884d468c92ebca35a47267b82c30c86b281052ae15cd25aa3499d9ad26b4c27fb1c8232a8ecce28bfbf656a44a099d5e28bea6ccc9523f01abe4
-  data.tar.gz: 8a72520c058109b0f63c41efe79bc91c174fffdca423940c8201a28e94bd4465d4110d9918ba39c505b0fc674ce6e819108d4837ff416f5befaaff9c2bdff4e9
+  metadata.gz: c2e5b5d80869c5c90b234af5edbb6d0c0e6118a3dc7191c2d7169209c7ae1505329fb18ea5e3803b045a527dd20b2909a9b2067732dfa030bbce0a931fc88954
+  data.tar.gz: 03d139a0f6ac0e967ad4f98685a6d8e513eab0b2022c7a0f7c6d27eca0da9b6441c30480a19acacf5c5f5f0ec762a7243331c3ee752b071015800a98043cad3b

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.4.1 (2022-10-15)
+
+- Added support for `json` format
+
+## 0.4.0 (2022-07-27)
+
+- Added support for pg_query 2
+- Switched to monotonic time
+- Dropped support for Ruby < 2.5
+
 ## 0.3.10 (2021-03-25)
 
 - Require pg_query < 2

data/README.md

@@ -2,7 +2,7 @@
 
 The automatic indexer for Postgres
 
-[Read about how it works](https://ankane.org/introducing-dexter)
+[Read about how it works](https://ankane.org/introducing-dexter) or [watch the talk](https://www.youtube.com/watch?v=Mni_1yTaNbE)
 
 [![Build Status](https://github.com/ankane/dexter/workflows/build/badge.svg?branch=master)](https://github.com/ankane/dexter/actions)
 
@@ -12,8 +12,8 @@ First, install [HypoPG](https://github.com/HypoPG/hypopg) on your database serve
 
 ```sh
 cd /tmp
-curl -L https://github.com/HypoPG/hypopg/archive/1.2.0.tar.gz | tar xz
-cd hypopg-1.2.0
+curl -L https://github.com/HypoPG/hypopg/archive/1.3.1.tar.gz | tar xz
+cd hypopg-1.3.1
 make
 make install # may need sudo
 ```
@@ -32,7 +32,7 @@ And install the command line tool with:
 gem install pgdexter
 ```
 
-The command line tool is also available as a [Linux package](guides/Linux.md).
+The command line tool is also available with [Docker](#docker), [Homebrew](#homebrew), or as a [Linux package](guides/Linux.md).
 
 ## How to Use
 
@@ -190,7 +190,37 @@ dexter --log-sql --log-level debug2
 
 ## Hosted Postgres
 
-Some hosted providers like Amazon RDS and Heroku do not support the HypoPG extension, which Dexter needs to run. See [how to use Dexter](guides/Hosted-Postgres.md) in these cases.
+Some hosted providers like Amazon RDS and Heroku do not support the HypoPG extension, which Dexter needs to run. See [how to use Dexter](guides/Hosted-Postgres.md) in these cases. To request the extension:
+
+- Amazon RDS - follow the instructions on [this page](https://aws.amazon.com/rds/postgresql/faqs/)
+- Google Cloud SQL - star the [feature request](https://issuetracker.google.com/issues/69250435)
+- DigitalOcean Managed Databases - follow the instructions on [this page](https://docs.digitalocean.com/products/databases/postgresql/details/supported-extensions/#supported-extensions)
+
+## Additional Installation Methods
+
+### Docker
+
+Get the [Docker image](https://hub.docker.com/r/ankane/dexter) with:
+
+```sh
+docker pull ankane/dexter
+```
+
+And run it with:
+
+```sh
+docker run -ti ankane/dexter <connection-options>
+```
+
+For databases on the host machine, use `host.docker.internal` as the hostname (on Linux, this requires Docker 20.04 and `--add-host=host.docker.internal:host-gateway`).
+
+### Homebrew
+
+With Homebrew, you can use:
+
+```sh
+brew install ankane/brew/dexter
+```
 
 ## Future Work
 
@@ -213,7 +243,7 @@ gem specific_install https://github.com/ankane/dexter.git
 
 ## Thanks
 
-This software wouldn’t be possible without [HypoPG](https://github.com/dalibo/hypopg), which allows you to create hypothetical indexes, and [pg_query](https://github.com/lfittl/pg_query), which allows you to parse and fingerprint queries. A big thanks to Dalibo and Lukas Fittl respectively.
+This software wouldn’t be possible without [HypoPG](https://github.com/HypoPG/hypopg), which allows you to create hypothetical indexes, and [pg_query](https://github.com/lfittl/pg_query), which allows you to parse and fingerprint queries. A big thanks to Dalibo and Lukas Fittl respectively.
 
 ## Research
 

data/lib/dexter/indexer.rb

@@ -240,6 +240,7 @@ module Dexter
     end
 
     def find_columns(plan)
+      plan = JSON.parse(plan.to_json)
       find_by_key(plan, "ColumnRef")
     end
 
@@ -248,20 +249,24 @@ module Dexter
     end
 
     def find_by_key(plan, key)
-      indexes = []
-      case plan
-      when Hash
-        plan.each do |k, v|
-          if k == key
-            indexes << v
-          else
-            indexes.concat(find_by_key(v, key))
+      result = []
+      queue = [plan]
+      while queue.any?
+        node = queue.pop
+        case node
+        when Hash
+          node.each do |k, v|
+            if k == key
+              result << v
+            elsif !v.nil?
+              queue << v
+            end
           end
+        when Array
+          queue.concat(node)
         end
-      when Array
-        indexes.concat(plan.flat_map { |v| find_by_key(v, key) })
       end
-      indexes
+      result
     end
 
     def hypo_indexes_from_plan(index_name_to_columns, plan, index_set)
@@ -483,10 +488,10 @@ module Dexter
               statement = String.new("CREATE INDEX CONCURRENTLY ON #{quote_ident(index[:table])} (#{index[:columns].map { |c| quote_ident(c) }.join(", ")})")
               statement << " TABLESPACE #{quote_ident(@tablespace)}" if @tablespace
               log "Creating index: #{statement}"
-              started_at = Time.now
+              started_at = monotonic_time
               begin
                 execute(statement)
-                log "Index created: #{((Time.now - started_at) * 1000).to_i} ms"
+                log "Index created: #{((monotonic_time - started_at) * 1000).to_i} ms"
               rescue PG::LockNotAvailable
                 log "Could not acquire lock: #{index[:table]}"
               end
@@ -498,6 +503,10 @@ module Dexter
       new_indexes
     end
 
+    def monotonic_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
     def conn
       @conn ||= begin
         # set connect timeout if none set

data/lib/dexter/json_log_parser.rb

@@ -0,0 +1,23 @@
+require "json"
+
+module Dexter
+  class JsonLogParser < LogParser
+    FIRST_LINE_REGEX = /\A.+/
+
+    def perform
+      @logfile.each_line do |line|
+        row = JSON.parse(line.chomp)
+        if (m = REGEX.match(row["message"]))
+          # replace first line with match
+          # needed for multiline queries
+          active_line = row["message"].sub(FIRST_LINE_REGEX, m[3])
+
+          add_parameters(active_line, row["detail"]) if row["detail"]
+          process_entry(active_line, m[1].to_f)
+        end
+      end
+    rescue JSON::ParserError => e
+      raise Dexter::Abort, "ERROR: #{e.message}"
+    end
+  end
+end

data/lib/dexter/processor.rb

@@ -13,6 +13,8 @@ module Dexter
           PgStatActivityParser.new(@indexer, @collector)
         elsif options[:input_format] == "csv"
           CsvLogParser.new(logfile, @collector)
+        elsif options[:input_format] == "json"
+          JsonLogParser.new(logfile, @collector)
         elsif options[:input_format] == "sql"
           SqlLogParser.new(logfile, @collector)
         else
@@ -62,7 +64,7 @@ module Dexter
     end
 
     def process_queries_without_lock
-      now = Time.now
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       min_checked_at = now - 3600 # don't recheck for an hour
       queries = []
       @collector.fetch_queries.each do |query|

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.3.10"
+  VERSION = "0.4.1"
 end

data/lib/dexter.rb

@@ -4,6 +4,7 @@ require "pg_query"
 require "slop"
 
 # stdlib
+require "json"
 require "set"
 require "time"
 
@@ -15,6 +16,7 @@ require "dexter/collector"
 require "dexter/indexer"
 require "dexter/log_parser"
 require "dexter/csv_log_parser"
+require "dexter/json_log_parser"
 require "dexter/pg_stat_activity_parser"
 require "dexter/sql_log_parser"
 require "dexter/processor"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.4.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-03-26 00:00:00.000000000 Z
+date: 2022-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -42,16 +42,16 @@ dependencies:
   name: pg_query
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '2.1'
 description:
 email: andrew@ankane.org
 executables:
@@ -68,6 +68,7 @@ files:
 - lib/dexter/collector.rb
 - lib/dexter/csv_log_parser.rb
 - lib/dexter/indexer.rb
+- lib/dexter/json_log_parser.rb
 - lib/dexter/log_parser.rb
 - lib/dexter/logging.rb
 - lib/dexter/pg_stat_activity_parser.rb
@@ -87,14 +88,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.2'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: The automatic indexer for Postgres