pgtk 0.18.2 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffaf4c44a63b471c11803b43d5e9774a25d7d261136f69e9d4a03bd99b4bd2f7
-  data.tar.gz: d3a3eee3b145b4edf8f4b3cf715efcb668b6d2b4c56574f8f23542484128b3a2
+  metadata.gz: 1cab54d9805f427ff6187a6cdec4fa2cd5424a0c4959191169e674eb2400a09a
+  data.tar.gz: 2d584fffabf7b943fc087d9bad1edb68296826b5b987e34725d5a252503f9f46
 SHA512:
-  metadata.gz: 30982738a66af8cc678f7833a8497ff77d4745bf2f54dda935b4929e2d19d413f6c30ff37228aae42ce98537fa78649b45d0811e2d393ff95f6d00f3b3c799b3
-  data.tar.gz: 030365d8a107dae67836b4f27cb6abef084c0d7bc9f60cbc1f8cc7e84b962cebd4b6eb3262d0a916b1a6bc8a0936b58a2d1196fb00c4e832621f6e04ece59bb3
+  metadata.gz: 6f3b1bd6f15acd85f50194b1aa9653f1b892b0dea6501dcacaa1c2cc38ccd3f609331fa3cccc7a3c79f86285861b28e77e87d6c84d4f7f9b81f272e938fed216
+  data.tar.gz: d583bfab4f9647f08235891899ab938dd4d778d49115c6a4bee4a72298810364fd419cdb271e145610001e8742f432ade4a6ed212b7bc74c62ce89caf0cc8f27

data/Gemfile.lock

@@ -26,7 +26,7 @@ GEM
       loog (~> 0.6)
       tago (~> 0.1)
     joined (0.4.0)
-    json (2.15.1)
+    json (2.15.2)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     logger (1.7.0)
@@ -45,7 +45,7 @@ GEM
     nokogiri (1.18.10-x86_64-linux-gnu)
       racc (~> 1.4)
     parallel (1.27.0)
-    parser (3.3.9.0)
+    parser (3.3.10.0)
       ast (~> 2.4.1)
       racc
     pg (1.6.2-arm64-darwin)
@@ -60,12 +60,12 @@ GEM
     racc (1.8.1)
     rack (3.2.3)
     rainbow (3.1.1)
-    rake (13.3.0)
+    rake (13.3.1)
     random-port (0.7.6)
       tago (~> 0.0)
     regexp_parser (2.11.3)
     rexml (3.4.4)
-    rubocop (1.81.6)
+    rubocop (1.81.7)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -102,7 +102,7 @@ GEM
     simplecov_json_formatter (0.1.4)
     slop (4.10.1)
     tago (0.3.0)
-    timeout (0.4.3)
+    timeout (0.4.4)
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
     unicode-emoji (4.1.0)

data/lib/pgtk/stash.rb

@@ -22,17 +22,46 @@ require_relative '../pgtk'
 # Copyright:: Copyright (c) 2019-2025 Yegor Bugayenko
 # License:: MIT
 class Pgtk::Stash
+  MODS = %w[INSERT DELETE UPDATE LOCK VACUUM TRANSACTION COMMIT ROLLBACK REINDEX TRUNCATE CREATE ALTER DROP SET].freeze
+  MODS_RE = Regexp.new("(^|\\s)(#{MODS.join('|')})(\\s|$)")
+
+  ALTS = ['UPDATE', 'INSERT INTO', 'DELETE FROM', 'TRUNCATE', 'ALTER TABLE', 'DROP TABLE'].freeze
+  ALTS_RE = Regexp.new("(?<=^|\\s)(?:#{ALTS.join('|')})\\s([a-z]+)(?=[^a-z]|$)")
+
+  private_constant :MODS, :ALTS, :MODS_RE, :ALTS_RE
+
   # Initialize a new Stash with query caching.
   #
   # @param [Object] pgsql PostgreSQL connection object
   # @param [Hash] stash Optional existing stash to use (default: new empty stash)
+  # @option [Hash] queries Internal cache data (default: {})
+  # @option [Hash] tables Internal cache data (default: {})
+  # @option [Concurrent::ReentrantReadWriteLock] entrance Lock for write internal state
+  # @option [Concurrent::AtomicBoolean] start_refresher Latch for start timers once
+  # @param [Integer] refresh Interval in seconds for recalculate stale queries
+  # @param [Integer] top Number of queries to recalculate
+  # @param [Integer] threads Number of threads in threadpool
   # @param [Loog] loog Logger for debugging (default: null logger)
-  def initialize(pgsql, stash = {})
+  def initialize(
+    pgsql,
+    stash = {
+      queries: {},
+      tables: {},
+      entrance: Concurrent::ReentrantReadWriteLock.new,
+      start_refresher: Concurrent::AtomicBoolean.new(false)
+    },
+    refresh: 5,
+    top: 100,
+    threads: 5,
+    loog: Loog::NULL
+  )
     @pgsql = pgsql
     @stash = stash
-    @stash[:queries] ||= {}
-    @stash[:tables] ||= {}
-    @entrance = Concurrent::ReentrantReadWriteLock.new
+    @entrance = stash[:entrance]
+    @refresh = refresh
+    @top = top
+    @threads = threads
+    @loog = loog
   end
 
   # Execute a SQL query with optional caching.
@@ -44,30 +73,29 @@ class Pgtk::Stash
   # @param [Integer] result Should be 0 for text results, 1 for binary
   # @return [PG::Result] Query result
   def exec(query, params = [], result = 0)
-    mods = %w[INSERT DELETE UPDATE LOCK VACUUM TRANSACTION COMMIT ROLLBACK REINDEX TRUNCATE CREATE ALTER DROP SET]
-    alts = ['UPDATE', 'INSERT INTO', 'DELETE FROM', 'TRUNCATE', 'ALTER TABLE', 'DROP TABLE']
     pure = (query.is_a?(Array) ? query.join(' ') : query).gsub(/\s+/, ' ').strip
-    if Regexp.new("(^|\\s)(#{mods.join('|')})(\\s|$)").match?(pure) || /(^|\s)pg_[a-z_]+\(/.match?(pure)
-      tables = pure.scan(Regexp.new("(?<=^|\\s)(?:#{alts.join('|')})\\s([a-z]+)(?=[^a-z]|$)")).map(&:first).uniq
+    if MODS_RE.match?(pure) || /(^|\s)pg_[a-z_]+\(/.match?(pure)
+      tables = pure.scan(ALTS_RE).map(&:first).uniq
       ret = @pgsql.exec(pure, params, result)
       @entrance.with_write_lock do
         tables.each do |t|
           @stash[:tables][t]&.each do |q|
-            @stash[:queries].delete(q)
+            @stash[:queries][q].each_key do |key|
+              @stash[:queries][q][key]['stale'] = true
+            end
           end
-          @stash[:tables].delete(t)
         end
       end
     else
       key = params.map(&:to_s).join(' -*&%^- ')
       @entrance.with_write_lock { @stash[:queries][pure] ||= {} }
-      ret = @stash[:queries][pure][key]
-      if ret.nil?
+      ret = @stash.dig(:queries, pure, key, 'ret')
+      if ret.nil? || @stash.dig(:queries, pure, key, 'stale')
         ret = @pgsql.exec(pure, params, result)
         unless pure.include?(' NOW() ')
           @entrance.with_write_lock do
             @stash[:queries][pure] ||= {}
-            @stash[:queries][pure][key] = ret
+            @stash[:queries][pure][key] = { 'ret' => ret, 'params' => params, 'result' => result }
             tables = pure.scan(/(?<=^|\s)(?:FROM|JOIN) ([a-z_]+)(?=\s|$)/).map(&:first).uniq
             tables.each do |t|
               @stash[:tables][t] = [] if @stash[:tables][t].nil?
@@ -77,6 +105,7 @@ class Pgtk::Stash
           end
         end
       end
+      count(pure, key)
     end
     ret
   end
@@ -89,7 +118,13 @@ class Pgtk::Stash
   # @return [Object] The result of the block
   def transaction
     @pgsql.transaction do |t|
-      yield Pgtk::Stash.new(t, @stash)
+      yield Pgtk::Stash.new(
+        t, @stash,
+        refresh: @refresh,
+        top: @top,
+        threads: @threads,
+        loog: @loog
+      )
     end
   end
 
@@ -98,7 +133,14 @@ class Pgtk::Stash
   # @param args Arguments to pass to the underlying pool's start method
   # @return [Pgtk::Stash] A new stash that shares the same cache
   def start(*args)
-    Pgtk::Stash.new(@pgsql.start(*args), @stash)
+    start_refresher
+    Pgtk::Stash.new(
+      @pgsql.start(*args), @stash,
+      refresh: @refresh,
+      top: @top,
+      threads: @threads,
+      loog: @loog
+    )
   end
 
   # Get the PostgreSQL server version.
@@ -107,4 +149,58 @@ class Pgtk::Stash
   def version
     @pgsql.version
   end
+
+  # Get statistics on the most used queries
+  #
+  # @return [Array<Array<String, Integer>>] Array of query and hits in desc hits order
+  def stats
+    @stash[:queries].map { |k, v| [k.dup, v.values.sum { |vv| vv['count'] }] }.sort_by { -_1[1] }
+  end
+
+  private
+
+  def count(query, key)
+    @entrance.with_write_lock do
+      @stash[:queries][query][key]['count'] ||= 0
+      @stash[:queries][query][key]['count'] += 1
+    end
+  end
+
+  def start_refresher
+    raise 'Cannot start cache refresh multiple times on same cache data' unless @stash[:start_refresher].make_true
+    Concurrent::FixedThreadPool.new(@threads).then do |threadpool|
+      Concurrent::TimerTask.execute(execution_interval: 24 * 60 * 60, executor: threadpool) do
+        @entrance.with_write_lock do
+          @stash[:queries].each_key do |q|
+            @stash[:queries][q].each_key do |k|
+              @stash[:queries][q][k]['count'] = 0
+            end
+          end
+        end
+      end
+      Concurrent::TimerTask.execute(execution_interval: @refresh, executor: threadpool) do
+        @stash[:queries]
+          .map { |k, v| [k, v.values.sum { |vv| vv['count'] }, v.values.any? { |vv| vv['stale'] }] }
+          .select { _1[2] }
+          .sort_by { -_1[1] }
+          .first(@top)
+          .each do |a|
+          q = a[0]
+          @stash[:queries][q].each_key do |k|
+            next unless @stash[:queries][q][k]['stale']
+            threadpool.post do
+              params = @stash[:queries][q][k]['params']
+              result = @stash[:queries][q][k]['result']
+              ret = @pgsql.exec(q, params, result)
+              @entrance.with_write_lock do
+                @stash[:queries][q] ||= {}
+                @stash[:queries][q][k] = { 'ret' => ret, 'params' => params, 'result' => result, 'count' => 1 }
+              end
+            end
+          end
+        end
+      end
+    end
+    nil
+  end
 end

data/lib/pgtk/version.rb

@@ -11,5 +11,5 @@ require_relative '../pgtk'
 # License:: MIT
 module Pgtk
   # Current version of the library.
-  VERSION = '0.18.2'
+  VERSION = '0.19.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgtk
 version: !ruby/object:Gem::Version
-  version: 0.18.2
+  version: 0.19.0
 platform: ruby
 authors:
 - Yegor Bugayenko