pgdexter 0.5.2 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e51edc18f4049b9ca811e4662248aa7a23e748b000975a87fa4a431f49ec632b
-  data.tar.gz: f3aa243431e0aab1d84d4cbc48ee6a329581acedc7a9c70e2a13e7bb0e15b529
+  metadata.gz: 037ac99e7dff1ac6edb6fddf8e85ae8357c6af25bc83dde1e5281f87c247ec4c
+  data.tar.gz: 1256486b21d6d6f91186a1a4e3ca0c1c39c7e318b26335d5cd86f82252b7ec42
 SHA512:
-  metadata.gz: 67b04a3c336c3bec239490a386825153699c646a01f5fe30d84b42445695f2431b1545bace72c5b955e03e92ad531810b5a73971d559abb06f175a1965454b89
-  data.tar.gz: 63691b01566b54056201c2d0a5c73a8153252b7c2ce4619983feb3b20e58d49d4a738b210be61d46231d3b5c78c1787664c625c0eb0216ee0d9ba19dcc8a7621
+  metadata.gz: 4bc5d0355a53b867da9710a23edc7a898bd23f112ee93beee246b04584175609ac750f9819cb1be041b26f36db407363601e57a5cb8effb49214148c9ef688cb
+  data.tar.gz: a9019542cfefe04789426309834f7688cc59b8dea8d2d4f3d4559b349ae5bb72bf56de4bcd7ee54ec2483f41df85edec8dfd62f4e2e4db79d30117a874a02fd8

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.5.4 (2024-04-03)
+
+- Fixed issue with processing over 500 query fingerprints (introduced in 0.5.3)
+- Require google-protobuf < 4
+
+## 0.5.3 (2024-03-05)
+
+- Fixed error with hypothetical index limit
+- Fixed error with foreign tables
+
 ## 0.5.2 (2024-01-10)
 
 - Added Docker image for `linux/arm64`

data/lib/dexter/indexer.rb

@@ -96,10 +96,16 @@ module Dexter
       analyze_tables(tables) if tables.any? && (@analyze || @log_level == "debug2")
 
       # create hypothetical indexes and explain queries
-      candidates = tables.any? ? create_hypothetical_indexes(queries.select(&:candidate_tables)) : {}
+      if tables.any?
+        # process in batches to prevent "hypopg: not more oid available" error
+        # https://hypopg.readthedocs.io/en/rel1_stable/usage.html#configuration
+        queries.select(&:candidate_tables).each_slice(500) do |batch|
+          create_hypothetical_indexes(batch)
+        end
+      end
 
       # see if new indexes were used and meet bar
-      new_indexes = determine_indexes(queries, candidates, tables)
+      new_indexes = determine_indexes(queries, tables)
 
       # display and create new indexes
       show_and_create_indexes(new_indexes, queries)
@@ -183,6 +189,8 @@ module Dexter
     def create_hypothetical_indexes(queries)
       candidates = {}
 
+      reset_hypothetical_indexes
+
       # get initial costs for queries
       calculate_plan(queries)
       explainable_queries = queries.select { |q| q.plans.any? && q.high_cost? }
@@ -228,7 +236,9 @@ module Dexter
         calculate_plan(explainable_queries)
       end
 
-      candidates
+      queries.each do |query|
+        query.candidates = candidates
+      end
     end
 
     def find_columns(plan)
@@ -282,9 +292,8 @@ module Dexter
       query_indexes
     end
 
-    def determine_indexes(queries, candidates, tables)
+    def determine_indexes(queries, tables)
       new_indexes = {}
-      index_name_to_columns = candidates.invert
 
       # filter out existing indexes
       # this must happen at end of process
@@ -313,11 +322,11 @@ module Dexter
           cost_savings2 = new_cost > 100 && new_cost2 < new_cost * savings_ratio
 
           key = cost_savings2 ? 2 : 1
-          query_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[key], index_set)
+          query_indexes = hypo_indexes_from_plan(query.candidates, query.plans[key], index_set)
 
           # likely a bad suggestion, so try single column
           if cost_savings2 && query_indexes.size > 1
-            query_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[1], index_set)
+            query_indexes = hypo_indexes_from_plan(query.candidates, query.plans[1], index_set)
             cost_savings2 = false
           end
 
@@ -390,8 +399,8 @@ module Dexter
 
           # TODO optimize
           if @log_level.start_with?("debug")
-            query.pass1_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[1], index_set)
-            query.pass2_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[2], index_set)
+            query.pass1_indexes = hypo_indexes_from_plan(query.candidates, query.plans[1], index_set)
+            query.pass2_indexes = hypo_indexes_from_plan(query.candidates, query.plans[2], index_set)
           end
         end
       end
@@ -595,7 +604,8 @@ module Dexter
       columns_by_table.each do |table, cols|
         # no reason to use btree index for json columns
         cols.reject { |c| ["json", "jsonb"].include?(c[:type]) }.permutation(n) do |col_set|
-          candidates[col_set] = create_hypothetical_index(table, col_set)
+          index_name = create_hypothetical_index(table, col_set)
+          candidates[index_name] = col_set
         end
       end
     end
@@ -612,6 +622,7 @@ module Dexter
           information_schema.tables
         WHERE
           table_catalog = current_database()
+          AND table_type IN ('BASE TABLE', 'VIEW')
       SQL
       result.map { |r| r["table_name"] }
     end

data/lib/dexter/query.rb

@@ -2,7 +2,7 @@ module Dexter
   class Query
     attr_reader :statement, :fingerprint, :plans
     attr_writer :tables
-    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes, :pass3_indexes, :candidate_tables, :tables_from_views
+    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes, :pass3_indexes, :candidate_tables, :tables_from_views, :candidates
 
     def initialize(statement, fingerprint = nil)
       @statement = statement

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.5.2"
+  VERSION = "0.5.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.4
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-01-10 00:00:00.000000000 Z
+date: 2024-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: google-protobuf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement