ruby-pg-extras 5.6.18 → 5.8.0

data/spec/diagnose_data_spec.rb

@@ -59,6 +59,12 @@ describe RubyPgExtras::DiagnoseData do
             { table: "posts", column_name: "topic_id" },
           ]
         }
+
+        expect(RubyPgExtras)
+          .to receive(:update_stats)
+          .with(in_format: :hash)
+          .twice
+          .and_return([])
       end
 
       it "works" do
@@ -76,4 +82,250 @@ describe RubyPgExtras::DiagnoseData do
       end
     end
   end
+
+  describe "#new_page_updates" do
+    let(:diagnose_data) { described_class.new }
+
+    it "reports tables exceeding the update sample and new-page thresholds" do
+      allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return(
+        [
+          {
+            "table" => "orders",
+            "fillfactor" => "100",
+            "total_updates" => "10000",
+            "new_page_updates" => "2500",
+            "new_page_pct" => "25.00",
+            "hot_given_same_page_pct" => "90.00",
+          },
+          {
+            "table" => "users",
+            "fillfactor" => "80",
+            "total_updates" => "9999",
+            "new_page_updates" => "3000",
+            "new_page_pct" => "30.00",
+            "hot_given_same_page_pct" => "95.00",
+          },
+        ],
+      )
+
+      result = diagnose_data.send(:new_page_updates)
+
+      expect(result).to eq(
+        ok: false,
+        message: <<~MESSAGE.strip,
+          High new-page update ratios detected:
+
+          'orders':
+            new-page updates: 25.00% (2500 of 10000)
+            HOT among same-page updates: 90.00%
+            fillfactor: 100
+
+          A high new-page ratio means many successor tuple versions were placed on another heap page and therefore could not be HOT. This commonly indicates insufficient reusable space on the original page. `n_tup_newpage_upd` records that placement directly; it does not identify the underlying reason or whether the update would otherwise have been HOT-eligible. Investigate page-space pressure, row growth, long-lived transactions, large update batches, and whether a lower table fillfactor is appropriate.
+
+          The HOT-among-same-page percentage provides additional context: a low value suggests indexed-column changes are preventing HOT on updates that did stay on the same page, so changing fillfactor alone may not resolve the issue.
+
+          These counters are cumulative; compare their deltas before and after a change.
+        MESSAGE
+      )
+    end
+
+    it "does not report tables below either threshold" do
+      allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return(
+        [
+          {
+            "table" => "orders",
+            "total_updates" => "10000",
+            "new_page_pct" => "19.99",
+          },
+          {
+            "table" => "users",
+            "total_updates" => "9999",
+            "new_page_pct" => "25.00",
+          },
+        ],
+      )
+
+      expect(diagnose_data.send(:new_page_updates)).to eq(
+        ok: true,
+        message: "No tables with a high new-page update ratio detected.",
+      )
+    end
+
+    it "allows overriding the thresholds with environment variables" do
+      allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return(
+        [
+          {
+            "table" => "orders",
+            "fillfactor" => "100",
+            "total_updates" => "500",
+            "new_page_updates" => "75",
+            "new_page_pct" => "15.00",
+            "hot_given_same_page_pct" => "90.00",
+          },
+        ],
+      )
+      original_max_percent = ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT"]
+      original_min_sample = ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MIN_SAMPLE"]
+      ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT"] = "15"
+      ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MIN_SAMPLE"] = "500"
+
+      expect(diagnose_data.send(:new_page_updates).fetch(:ok)).to eq(false)
+    ensure
+      ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT"] = original_max_percent
+      ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MIN_SAMPLE"] = original_min_sample
+    end
+
+    it "skips the check when update_stats returns the legacy breakdown" do
+      allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return(
+        [
+          {
+            "table" => "orders",
+            "total_updates" => "10000",
+            "hot_updates" => "5000",
+            "hot_pct" => "50.00",
+          },
+        ],
+      )
+
+      expect(diagnose_data.send(:new_page_updates)).to eq(
+        ok: true,
+        message: "New-page update analysis requires PostgreSQL 16 or newer.",
+      )
+    end
+  end
+
+  describe "#low_hot_same_page" do
+    let(:diagnose_data) { described_class.new }
+
+    it "reports tables below the HOT-among-same-page threshold" do
+      allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return(
+        [
+          {
+            "table" => "sessions",
+            "fillfactor" => "100",
+            "total_updates" => "10000",
+            "same_page_pct" => "95.61",
+            "new_page_pct" => "4.39",
+            "hot_given_same_page_pct" => "0.0",
+          },
+          {
+            "table" => "orders",
+            "fillfactor" => "80",
+            "total_updates" => "10000",
+            "same_page_pct" => "90.00",
+            "new_page_pct" => "10.00",
+            "hot_given_same_page_pct" => "10.00",
+          },
+          {
+            "table" => "users",
+            "fillfactor" => "100",
+            "total_updates" => "9999",
+            "same_page_pct" => "99.00",
+            "new_page_pct" => "1.00",
+            "hot_given_same_page_pct" => "0.0",
+          },
+        ],
+      )
+
+      result = diagnose_data.send(:low_hot_same_page)
+
+      expect(result).to eq(
+        ok: false,
+        message: <<~MESSAGE.strip,
+          Low HOT-among-same-page update ratios detected:
+
+          'sessions':
+            HOT among same-page updates: 0.0%
+            same-page updates: 95.61%
+            new-page updates: 4.39%
+            fillfactor: 100
+
+          A low HOT-among-same-page ratio means updates that stayed on the original heap page still could not be HOT. That usually means those updates modified indexed columns. Review which columns your application updates and which indexes cover them; removing or adjusting indexes on frequently updated columns (or avoiding updating those columns) can restore HOT updates and reduce index and vacuum overhead.
+
+          These counters are cumulative; compare their deltas before and after a change.
+        MESSAGE
+      )
+    end
+
+    it "does not report tables at or above the threshold" do
+      allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return(
+        [
+          {
+            "table" => "orders",
+            "total_updates" => "10000",
+            "hot_given_same_page_pct" => "10.00",
+          },
+          {
+            "table" => "users",
+            "total_updates" => "9999",
+            "hot_given_same_page_pct" => "0.0",
+          },
+        ],
+      )
+
+      expect(diagnose_data.send(:low_hot_same_page)).to eq(
+        ok: true,
+        message: "No tables with a low HOT-among-same-page update ratio detected.",
+      )
+    end
+
+    it "skips tables with a NULL HOT-among-same-page ratio" do
+      allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return(
+        [
+          {
+            "table" => "orders",
+            "total_updates" => "10000",
+            "hot_given_same_page_pct" => nil,
+          },
+        ],
+      )
+
+      expect(diagnose_data.send(:low_hot_same_page)).to eq(
+        ok: true,
+        message: "No tables with a low HOT-among-same-page update ratio detected.",
+      )
+    end
+
+    it "allows overriding the thresholds with environment variables" do
+      allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return(
+        [
+          {
+            "table" => "sessions",
+            "fillfactor" => "100",
+            "total_updates" => "500",
+            "same_page_pct" => "90.00",
+            "new_page_pct" => "10.00",
+            "hot_given_same_page_pct" => "4.00",
+          },
+        ],
+      )
+      original_min_percent = ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_PERCENT"]
+      original_min_sample = ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_SAMPLE"]
+      ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_PERCENT"] = "5"
+      ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_SAMPLE"] = "500"
+
+      expect(diagnose_data.send(:low_hot_same_page).fetch(:ok)).to eq(false)
+    ensure
+      ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_PERCENT"] = original_min_percent
+      ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_SAMPLE"] = original_min_sample
+    end
+
+    it "skips the check when update_stats returns the legacy breakdown" do
+      allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return(
+        [
+          {
+            "table" => "orders",
+            "total_updates" => "10000",
+            "hot_updates" => "5000",
+            "hot_pct" => "50.00",
+          },
+        ],
+      )
+
+      expect(diagnose_data.send(:low_hot_same_page)).to eq(
+        ok: true,
+        message: "HOT-among-same-page update analysis requires PostgreSQL 16 or newer.",
+      )
+    end
+  end
 end

data/spec/index_info_spec.rb

@@ -12,8 +12,38 @@ describe RubyPgExtras::IndexInfo do
       before do
         expect(RubyPgExtras).to receive(:indexes) {
           [
-            { "schemaname" => "public", "indexname" => "index_users_on_api_auth_token", "tablename" => "users", "columns" => "api_auth_token, column2" },
-            { "schemaname" => "public", "indexname" => "index_teams_on_slack_id", "tablename" => "teams", "columns" => "slack_id" },
+            {
+              "schemaname" => "public",
+              "indexname" => "index_users_on_api_auth_token",
+              "tablename" => "users",
+              "columns" => "api_auth_token, column2",
+              "columns_json" => '["api_auth_token","column2"]',
+              "key_columns" => "api_auth_token, column2",
+              "key_column_names" => '["api_auth_token","column2"]',
+              "included_columns" => "",
+              "included_columns_json" => "[]",
+              "index_method" => "btree",
+              "is_unique" => "f",
+              "is_primary" => "f",
+              "is_partial" => "f",
+              "predicate" => nil,
+            },
+            {
+              "schemaname" => "public",
+              "indexname" => "index_teams_on_slack_id",
+              "tablename" => "teams",
+              "columns" => "slack_id",
+              "columns_json" => '["slack_id"]',
+              "key_columns" => "slack_id",
+              "key_column_names" => '["slack_id"]',
+              "included_columns" => "external_id",
+              "included_columns_json" => '["external_id"]',
+              "index_method" => "btree",
+              "is_unique" => "t",
+              "is_primary" => "f",
+              "is_partial" => "t",
+              "predicate" => "external_id IS NOT NULL",
+            },
           ]
         }
 
@@ -43,6 +73,23 @@ describe RubyPgExtras::IndexInfo do
           RubyPgExtras::IndexInfoPrint.call(result)
         }.not_to raise_error
       end
+
+      it "returns structured index metadata" do
+        # The service keeps display columns and logical key columns as separate arrays.
+        expect(result).to include(
+          include(
+            index_name: "index_teams_on_slack_id",
+            columns: ["slack_id"],
+            key_columns: ["slack_id"],
+            included_columns: ["external_id"],
+            index_method: "btree",
+            unique: true,
+            primary: false,
+            partial: true,
+            predicate: "external_id IS NOT NULL",
+          )
+        )
+      end
     end
 
     context "real data" do
@@ -51,6 +98,68 @@ describe RubyPgExtras::IndexInfo do
           RubyPgExtras::IndexInfoPrint.call(result)
         }.not_to raise_error
       end
+
+      it "handles expression indexes without splitting on parentheses" do
+        index = result.find { |el| el.fetch(:index_name) == "index_users_on_lower_email" }
+
+        # Expression indexes can contain nested parentheses, so DDL string splitting is unsafe.
+        expect(index.fetch(:columns).first).to include("lower")
+      end
+
+      it "keeps partial index predicates separate from columns" do
+        index = result.find { |el| el.fetch(:index_name) == "index_users_on_email_partial" }
+
+        # Partial index predicates describe row coverage and should not be mixed into columns.
+        expect(index).to include(
+          columns: ["email"],
+          key_columns: ["email"],
+          partial: true,
+        )
+        expect(index.fetch(:predicate)).to include("customer_id IS NOT NULL")
+      end
+
+      it "keeps operator classes in display columns and clean key column names" do
+        index = result.find { |el| el.fetch(:index_name) == "index_users_on_email_pattern" }
+
+        # Opclasses affect index behavior, but FK checks still need the plain column name.
+        expect(index.fetch(:columns)).to eq(["email text_pattern_ops"])
+        expect(index.fetch(:key_columns)).to eq(["email"])
+      end
+
+      it "keeps sort order in display columns and clean key column names" do
+        index = result.find { |el| el.fetch(:index_name) == "index_posts_on_external_id_desc" }
+
+        # Sort/null options are display metadata, not part of the logical column name.
+        expect(index.fetch(:columns)).to eq(["external_id DESC NULLS LAST"])
+        expect(index.fetch(:key_columns)).to eq(["external_id"])
+      end
+
+      it "keeps per-position options on composite indexes" do
+        index = result.find { |el| el.fetch(:index_name) == "index_posts_on_user_id_desc_and_title_pattern" }
+
+        # Each position can have distinct options, so the query must pair metadata by position.
+        expect(index.fetch(:columns)).to eq(["user_id DESC NULLS FIRST", 'title COLLATE "C" text_pattern_ops'])
+        expect(index.fetch(:key_columns)).to eq(["user_id", "title"])
+      end
+
+      it "keeps collations in display columns and clean key column names" do
+        index = result.find { |el| el.fetch(:index_name) == "index_users_on_email_collate_c" }
+
+        # Explicit non-default collations should remain visible in index_info output.
+        expect(index.fetch(:columns).first).to include('COLLATE "C"')
+        expect(index.fetch(:key_columns)).to eq(["email"])
+      end
+
+      it "keeps included columns separate from key columns" do
+        index = result.find { |el| el.fetch(:index_name) == "index_users_on_email_include_customer_id" }
+
+        # INCLUDE columns can support index-only scans but are not search key columns.
+        expect(index).to include(
+          columns: ["email"],
+          key_columns: ["email"],
+          included_columns: ["customer_id"],
+        )
+      end
     end
   end
 end

data/spec/missing_fk_indexes_spec.rb

@@ -7,13 +7,18 @@ describe "missing_fk_indexes" do
   it "detects missing indexes for all tables" do
     result = RubyPgExtras.missing_fk_indexes(in_format: :hash)
     expect(result).to match_array([
-      { table: "users", column_name: "company_id" },
+      { table: "expression_not_null_partial_indexed_posts", column_name: "topic_id" },
+      { table: "expression_indexed_posts", column_name: "topic_id" },
+      { table: "null_partial_indexed_posts", column_name: "topic_id" },
+      { table: "partial_indexed_posts", column_name: "topic_id" },
       { table: "posts", column_name: "topic_id" },
     ])
   end
 
-  it "detects missing indexes for specific table" do
+  it "detects foreign keys that are only indexed after another column" do
     result = RubyPgExtras.missing_fk_indexes(args: { table_name: "posts" }, in_format: :hash)
+
+    # posts.topic_id exists in index_posts_on_user_id, but not as the leftmost key column.
     expect(result).to eq([
       { table: "posts", column_name: "topic_id" },
     ])
@@ -25,8 +30,11 @@ describe "missing_fk_indexes" do
       in_format: :hash
     )
 
-    expect(result).to eq([
-      { table: "users", column_name: "company_id" },
+    expect(result).to match_array([
+      { table: "expression_not_null_partial_indexed_posts", column_name: "topic_id" },
+      { table: "expression_indexed_posts", column_name: "topic_id" },
+      { table: "null_partial_indexed_posts", column_name: "topic_id" },
+      { table: "partial_indexed_posts", column_name: "topic_id" },
     ])
   end
 
@@ -36,8 +44,108 @@ describe "missing_fk_indexes" do
       in_format: :hash
     )
 
-    expect(result).to eq([
+    expect(result).to match_array([
+      { table: "expression_not_null_partial_indexed_posts", column_name: "topic_id" },
+      { table: "expression_indexed_posts", column_name: "topic_id" },
+      { table: "null_partial_indexed_posts", column_name: "topic_id" },
+      { table: "partial_indexed_posts", column_name: "topic_id" },
       { table: "posts", column_name: "topic_id" },
     ])
   end
+
+  it "does not flag foreign keys covered by sorted indexes" do
+    result = RubyPgExtras.missing_fk_indexes(args: { table_name: "users" }, in_format: :hash)
+
+    # users.company_id is leftmost in a sorted index, so it still supports FK checks.
+    expect(result).to eq([])
+  end
+
+  it "detects foreign keys covered only by an expression index" do
+    result = RubyPgExtras.missing_fk_indexes(args: { table_name: "expression_indexed_posts" }, in_format: :hash)
+
+    # Expression indexes do not support raw FK lookups such as WHERE topic_id = ?.
+    expect(result).to eq([
+      { table: "expression_indexed_posts", column_name: "topic_id" },
+    ])
+  end
+
+  it "detects foreign keys covered by an expression index with a not-null predicate" do
+    result = RubyPgExtras.missing_fk_indexes(args: { table_name: "expression_not_null_partial_indexed_posts" }, in_format: :hash)
+
+    # The predicate is compatible, but the index key is topic_id::text rather than raw topic_id.
+    expect(result).to eq([
+      { table: "expression_not_null_partial_indexed_posts", column_name: "topic_id" },
+    ])
+  end
+
+  it "detects foreign keys covered only by a partial index with an unrelated predicate" do
+    result = RubyPgExtras.missing_fk_indexes(args: { table_name: "partial_indexed_posts" }, in_format: :hash)
+
+    # WHERE id > 0 is unrelated to topic_id and does not guarantee coverage for every FK value.
+    expect(result).to eq([
+      { table: "partial_indexed_posts", column_name: "topic_id" },
+    ])
+  end
+
+  it "does not flag foreign keys covered by a not-null partial index" do
+    result = RubyPgExtras.missing_fk_indexes(args: { table_name: "not_null_partial_indexed_posts" }, in_format: :hash)
+
+    # FK checks only need non-null values, so WHERE topic_id IS NOT NULL still covers them.
+    expect(result).to eq([])
+  end
+
+  it "normalizes nested parentheses around not-null predicates" do
+    allow(RubyPgExtras).to receive(:indexes).and_return([
+      {
+        "tablename" => "stubbed_posts",
+        "columns" => "topic_id",
+        "key_column_names" => '["topic_id"]',
+        "is_partial" => "t",
+        "predicate" => "((topic_id IS NOT NULL))",
+      },
+    ])
+    allow(RubyPgExtras).to receive(:foreign_keys).and_return([
+      { "table_name" => "stubbed_posts", "column_name" => "topic_id" },
+    ])
+
+    result = RubyPgExtras.missing_fk_indexes(args: { table_name: "stubbed_posts" }, in_format: :hash)
+
+    expect(result).to eq([])
+  end
+
+  it "detects foreign keys covered only by a null partial index" do
+    result = RubyPgExtras.missing_fk_indexes(args: { table_name: "null_partial_indexed_posts" }, in_format: :hash)
+
+    # WHERE topic_id IS NULL cannot support FK lookups for concrete topic_id values.
+    expect(result).to eq([
+      { table: "null_partial_indexed_posts", column_name: "topic_id" },
+    ])
+  end
+
+  it "does not flag foreign keys covered by indexes with included columns" do
+    result = RubyPgExtras.missing_fk_indexes(args: { table_name: "included_indexed_posts" }, in_format: :hash)
+
+    # INCLUDE columns are ignored for key matching; topic_id is still the leftmost key column.
+    expect(result).to eq([])
+  end
+
+  it "does not flag foreign keys covered by indexes with operator classes" do
+    result = RubyPgExtras.missing_fk_indexes(args: { table_name: "opclass_indexed_codes" }, in_format: :hash)
+
+    # The display column includes text_pattern_ops, but the logical key column remains code.
+    expect(result).to eq([])
+  end
+
+  it "does not flag foreign keys covered by indexes with collations" do
+    result = RubyPgExtras.missing_fk_indexes(args: { table_name: "collated_indexed_codes" }, in_format: :hash)
+
+    # The display column includes COLLATE "C", but the logical key column remains code.
+    expect(result).to eq([])
+  end
+
+  it "does not flag foreign keys covered by sorted indexes on their own table" do
+    result = RubyPgExtras.missing_fk_indexes(args: { table_name: "sorted_indexed_posts" }, in_format: :hash)
+
+    expect(result).to eq([])
+  end
 end

data/spec/smoke_spec.rb

@@ -58,6 +58,67 @@ describe RubyPgExtras do
     end
   end
 
+  describe "update_stats" do
+    it "returns a consistent HOT update breakdown" do
+      connection = RubyPgExtras.connection
+      server_version_num = connection.exec("SHOW server_version_num").to_a[0].values[0].to_i
+
+      # Keep this fixture local so every run starts with fresh statistics counters
+      # and a controlled fillfactor for producing HOT and non-HOT updates.
+      connection.exec("DROP TABLE IF EXISTS update_stats_test")
+      connection.exec(<<~SQL)
+        CREATE TABLE update_stats_test (
+          id INTEGER PRIMARY KEY,
+          value TEXT
+        ) WITH (fillfactor = 80)
+      SQL
+      connection.exec("INSERT INTO update_stats_test VALUES (1, 'before')")
+      # Updating the unindexed value column produces a HOT update.
+      connection.exec("UPDATE update_stats_test SET value = 'after' WHERE id = 1")
+      # Updating the primary key requires index maintenance, producing a non-HOT update.
+      connection.exec("UPDATE update_stats_test SET id = 2 WHERE id = 1")
+      # estimated_heap_bytes_per_live_row divides main-fork heap size by
+      # pg_class.reltuples, which is only populated after ANALYZE (or VACUUM).
+      connection.exec("ANALYZE update_stats_test")
+
+      row = nil
+      # PostgreSQL publishes cumulative statistics asynchronously, particularly
+      # on older supported versions, so poll until both updates are visible.
+      20.times do
+        row = RubyPgExtras.update_stats(
+          args: { schema: "public" },
+          in_format: :hash,
+        ).find { |result| result["table"] == "update_stats_test" }
+        break if row && row["total_updates"].to_i == 2
+
+        sleep 0.1
+      end
+
+      expect(row).not_to be_nil
+      expect(row["fillfactor"].to_i).to eq(80)
+      expect(row["estimated_heap_bytes_per_live_row"].to_i).to be > 0
+      expect(row["total_updates"].to_i).to eq(2)
+      expect(row["hot_updates"].to_i).to eq(1)
+
+      if server_version_num >= 160000
+        expect(row["same_page_non_hot_updates"].to_i).to eq(1)
+        expect(row["new_page_updates"].to_i).to eq(0)
+        expect(row["total_updates"].to_i).to eq(
+          row["hot_updates"].to_i +
+          row["same_page_non_hot_updates"].to_i +
+          row["new_page_updates"].to_i,
+        )
+      else
+        expect(row["non_hot_updates"].to_i).to eq(1)
+        expect(row["total_updates"].to_i).to eq(
+          row["hot_updates"].to_i + row["non_hot_updates"].to_i,
+        )
+      end
+    ensure
+      connection&.exec("DROP TABLE IF EXISTS update_stats_test")
+    end
+  end
+
   describe "#database_url=" do
     it "setting custom database URL works" do
       RubyPgExtras.database_url = ENV.fetch("DATABASE_URL")