pgdexter 0.5.1 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c68746f6134f6603c5549b561886b45ee3432df5d1e07e65a5907559901d333a
-  data.tar.gz: 699c2744f3e2c9a79f8fa9ca6ae7a77586421243cbf7a1f92642f76724cc0cb0
+  metadata.gz: d40fb941685400a6a158502895da0d8b93d94db03da0ca4551714aea36e3dae3
+  data.tar.gz: 8b99de845f8d6f2e455e98c3505e689a0a55dac69f7c08ba6ebebabeeb1ddd63
 SHA512:
-  metadata.gz: 343bc52539ef09541fd0774667034ea02fb88b0e4f820e52c86d77df46d059c253576c22272b91a407b764965f66c554278154da6ab923249ce9979ab3a5aed0
-  data.tar.gz: a9b86931b4fc58d2b89c87f46f5008dfd47274cc39f096ec2b5143bea8858e29a786e387f9ccc29fe74475b5b0317e9d348d36f2a686cb19457ee79179a270f5
+  metadata.gz: b2b25419796e276074f5a49b2d109be72789ddf8a5c7c5cf29ba55900ac79c323e55b06bf5ab20dde4d34e4cf1e752e7362d23481d21d19a8b068a360562d13f
+  data.tar.gz: 382792e95461d718489d36c71e1ba01940f4b1d76de1b80527f0a4a19fcc90c215366645640970271011d5ad2c264051be57e62684e3278248d52bfaae983cd2

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.5.3 (2024-03-05)
+
+- Fixed error with hypothetical index limit
+- Fixed error with foreign tables
+
+## 0.5.2 (2024-01-10)
+
+- Added Docker image for `linux/arm64`
+- Switched to `GENERIC_PLAN` for Postgres 16
+- Fixed error with `auto_explain`
+- Fixed warning with Ruby 3.3
+
 ## 0.5.1 (2023-05-27)
 
 - Fixed `JSON::NestingError`

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2017-2023 Andrew Kane
+Copyright (c) 2017-2024 Andrew Kane
 
 MIT License
 

data/README.md

@@ -4,7 +4,7 @@ The automatic indexer for Postgres
 
 [Read about how it works](https://ankane.org/introducing-dexter) or [watch the talk](https://www.youtube.com/watch?v=Mni_1yTaNbE)
 
-[![Build Status](https://github.com/ankane/dexter/workflows/build/badge.svg?branch=master)](https://github.com/ankane/dexter/actions)
+[![Build Status](https://github.com/ankane/dexter/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/dexter/actions)
 
 ## Installation
 

data/lib/dexter/indexer.rb

@@ -96,10 +96,16 @@ module Dexter
       analyze_tables(tables) if tables.any? && (@analyze || @log_level == "debug2")
 
       # create hypothetical indexes and explain queries
-      candidates = tables.any? ? create_hypothetical_indexes(queries.select(&:candidate_tables)) : {}
+      if tables.any?
+        # process in batches to prevent "hypopg: not more oid available" error
+        # https://hypopg.readthedocs.io/en/rel1_stable/usage.html#configuration
+        queries.select(&:candidate_tables).each_slice(500) do |batch|
+          create_hypothetical_indexes(batch)
+        end
+      end
 
       # see if new indexes were used and meet bar
-      new_indexes = determine_indexes(queries, candidates, tables)
+      new_indexes = determine_indexes(queries, tables)
 
       # display and create new indexes
       show_and_create_indexes(new_indexes, queries)
@@ -185,7 +191,7 @@ module Dexter
 
       # get initial costs for queries
       calculate_plan(queries)
-      explainable_queries = queries.select { |q| q.explainable? && q.high_cost? }
+      explainable_queries = queries.select { |q| q.plans.any? && q.high_cost? }
 
       # filter tables for performance
       tables = Set.new(explainable_queries.flat_map(&:tables))
@@ -228,7 +234,9 @@ module Dexter
         calculate_plan(explainable_queries)
       end
 
-      candidates
+      queries.each do |query|
+        query.candidates = candidates
+      end
     end
 
     def find_columns(plan)
@@ -282,9 +290,8 @@ module Dexter
       query_indexes
     end
 
-    def determine_indexes(queries, candidates, tables)
+    def determine_indexes(queries, tables)
       new_indexes = {}
-      index_name_to_columns = candidates.invert
 
       # filter out existing indexes
       # this must happen at end of process
@@ -313,11 +320,11 @@ module Dexter
           cost_savings2 = new_cost > 100 && new_cost2 < new_cost * savings_ratio
 
           key = cost_savings2 ? 2 : 1
-          query_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[key], index_set)
+          query_indexes = hypo_indexes_from_plan(query.candidates, query.plans[key], index_set)
 
           # likely a bad suggestion, so try single column
           if cost_savings2 && query_indexes.size > 1
-            query_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[1], index_set)
+            query_indexes = hypo_indexes_from_plan(query.candidates, query.plans[1], index_set)
             cost_savings2 = false
           end
 
@@ -390,8 +397,8 @@ module Dexter
 
           # TODO optimize
           if @log_level.start_with?("debug")
-            query.pass1_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[1], index_set)
-            query.pass2_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[2], index_set)
+            query.pass1_indexes = hypo_indexes_from_plan(query.candidates, query.plans[1], index_set)
+            query.pass2_indexes = hypo_indexes_from_plan(query.candidates, query.plans[2], index_set)
           end
         end
       end
@@ -521,8 +528,8 @@ module Dexter
       raise Dexter::Abort, e.message
     end
 
-    def execute(query, pretty: true, params: [])
-      # use exec_params instead of exec for security
+    def execute(query, pretty: true, params: [], use_exec: false)
+      # use exec_params instead of exec when possible for security
       #
       # Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
       # (There can be semicolons in it, but not more than one nonempty command.)
@@ -533,7 +540,11 @@ module Dexter
       log colorize("[sql] #{query}#{params.any? ? " /*#{params.to_json}*/" : ""}", :cyan) if @log_sql
 
       @mutex.synchronize do
-        conn.exec_params("#{query} /*dexter*/", params).to_a
+        if use_exec
+          conn.exec("#{query} /*dexter*/").to_a
+        else
+          conn.exec_params("#{query} /*dexter*/", params).to_a
+        end
       end
     end
 
@@ -543,7 +554,9 @@ module Dexter
 
       # try to EXPLAIN normalized queries
       # https://dev.to/yugabyte/explain-from-pgstatstatements-normalized-queries-how-to-always-get-the-generic-plan-in--5cfi
-      explain_normalized = query.include?("$1")
+      normalized = query.include?("$1")
+      generic_plan = normalized && server_version_num >= 160000
+      explain_normalized = normalized && !generic_plan
       if explain_normalized
         prepared_name = "dexter_prepared"
         execute("PREPARE #{prepared_name} AS #{safe_statement(query)}", pretty: false)
@@ -566,12 +579,14 @@ module Dexter
         end
       end
 
+      explain_prefix = generic_plan ? "GENERIC_PLAN, " : ""
+
       # strip semi-colons as another measure of defense
-      plan = JSON.parse(execute("EXPLAIN (FORMAT JSON) #{safe_statement(query)}", pretty: false).first["QUERY PLAN"], max_nesting: 1000).first["Plan"]
+      plan = JSON.parse(execute("EXPLAIN (#{explain_prefix}FORMAT JSON) #{safe_statement(query)}", pretty: false, use_exec: generic_plan).first["QUERY PLAN"], max_nesting: 1000).first["Plan"]
 
       if @log_explain
         # Pass format to prevent ANALYZE
-        puts execute("EXPLAIN (FORMAT TEXT) #{safe_statement(query)}", pretty: false).map { |r| r["QUERY PLAN"] }.join("\n")
+        puts execute("EXPLAIN (#{explain_prefix}FORMAT TEXT) #{safe_statement(query)}", pretty: false, use_exec: generic_plan).map { |r| r["QUERY PLAN"] }.join("\n")
       end
 
       plan
@@ -587,7 +602,8 @@ module Dexter
       columns_by_table.each do |table, cols|
         # no reason to use btree index for json columns
         cols.reject { |c| ["json", "jsonb"].include?(c[:type]) }.permutation(n) do |col_set|
-          candidates[col_set] = create_hypothetical_index(table, col_set)
+          index_name = create_hypothetical_index(table, col_set)
+          candidates[index_name] = col_set
         end
       end
     end
@@ -604,6 +620,7 @@ module Dexter
           information_schema.tables
         WHERE
           table_catalog = current_database()
+          AND table_type IN ('BASE TABLE', 'VIEW')
       SQL
       result.map { |r| r["table_name"] }
     end

data/lib/dexter/query.rb

@@ -2,7 +2,7 @@ module Dexter
   class Query
     attr_reader :statement, :fingerprint, :plans
     attr_writer :tables
-    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes, :pass3_indexes, :candidate_tables, :tables_from_views
+    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes, :pass3_indexes, :candidate_tables, :tables_from_views, :candidates
 
     def initialize(statement, fingerprint = nil)
       @statement = statement
@@ -31,7 +31,7 @@ module Dexter
     end
 
     def explainable?
-      plans.any?
+      plans.size >= 3
     end
 
     def costs

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.5.1"
+  VERSION = "0.5.3"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-27 00:00:00.000000000 Z
+date: 2024-03-06 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -96,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: The automatic indexer for Postgres