pgtk 0.22.1 → 0.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0dc255f274b10a4bc7fe0a0e4d6c2a68a918957e571a147062325baf77d91db1
-  data.tar.gz: 971da231e5ff704bf28d28f3655f170ab986327da714cd3f27bade9ff784c6bd
+  metadata.gz: b16d736e3b32a69aabf5a87fff3d4611f0af14c6c87e72fb7dc3e84d2ee027ee
+  data.tar.gz: 1c989c0af1b8ce28e325b0b9e146295e3cd24f02bb2cec0fee7ed2ac714ed2d6
 SHA512:
-  metadata.gz: d65bd86d962357d89353daa86da5cd10f255cd1840198217963fe854cc83805c44c5fbff48e4c109871a29d17f8d6c93932a2a186b4067d6be28348cc631e174
-  data.tar.gz: ed61c3ae252043956524564e4a4a6b087d9df684b83c509d3f92291d721e2840cdd21fbc0535bbd027bb6b436c3164518b710a8f7ecae68ff4b6a12980e83864
+  metadata.gz: a6dcaac89f9ea200f9030505df810fb4a0a2136387e98833960f830be10d19aa9f1b55f7c3f8e6fd447c8df30235233aafd8f7730bde99c8751cb910416e660f
+  data.tar.gz: 34e7b0682c858ce58397abf4c8694cdb849453f70d0d0fe1c220d96eb800a425602ae693a149b53b7b2a1fc1b8afafee3f9c72a494b1214393d21d2d1dbafebd

data/Gemfile.lock

@@ -26,7 +26,7 @@ GEM
       loog (~> 0.6)
       tago (~> 0.1)
     joined (0.4.0)
-    json (2.15.2)
+    json (2.16.0)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     logger (1.7.0)
@@ -52,7 +52,7 @@ GEM
     pg (1.6.2-x64-mingw-ucrt)
     pg (1.6.2-x86_64-linux)
     prism (1.6.0)
-    qbash (0.4.5)
+    qbash (0.4.7)
       backtrace (> 0)
       elapsed (> 0)
       loog (> 0)
@@ -76,7 +76,7 @@ GEM
       rubocop-ast (>= 1.47.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.47.1)
+    rubocop-ast (1.48.0)
       parser (>= 3.3.7.2)
       prism (~> 1.4)
     rubocop-minitest (0.38.2)
@@ -101,7 +101,7 @@ GEM
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
     slop (4.10.1)
-    tago (0.3.0)
+    tago (0.4.0)
     threads (0.4.1)
       backtrace (~> 0)
       concurrent-ruby (~> 1.0)

data/README.md

@@ -115,8 +115,8 @@ From inside your app you may find this class useful:
 
 ```ruby
 require 'pgtk/pool'
-pgsql = Pgtk::Pool.new(Pgtk::Wire::Yaml.new('config.yml'))
-pgsql.start!(5) # Start it with five simultaneous connections
+pgsql = Pgtk::Pool.new(Pgtk::Wire::Yaml.new('config.yml'), max: 5)
+pgsql.start! # Start it with five simultaneous connections
 ```
 
 You can also let it pick the connection parameters from the environment

data/lib/pgtk/impatient.rb

@@ -19,8 +19,8 @@ require_relative '../pgtk'
 # Basic usage:
 #
 #   # Create and configure a regular pool
-#   pool = Pgtk::Pool.new(wire)
-#   pool.start!(4)
+#   pool = Pgtk::Pool.new(wire, max: 4)
+#   pool.start!
 #
 #   # Wrap the pool in an impatient decorator with a 2-second timeout
 #   impatient = Pgtk::Impatient.new(pool, 2)
@@ -69,8 +69,8 @@ class Pgtk::Impatient
   end
 
   # Start a new connection pool with the given arguments.
-  def start!(*)
-    @pool.start!(*)
+  def start!
+    @pool.start!
   end
 
   # Get the version of PostgreSQL server.

data/lib/pgtk/pool.rb

@@ -7,6 +7,7 @@ require 'pg'
 require 'loog'
 require 'tago'
 require_relative '../pgtk'
+require_relative 'version'
 require_relative 'wire'
 
 # Pool provides a connection pool for PostgreSQL database connections.
@@ -29,8 +30,8 @@ require_relative 'wire'
 #   )
 #
 #   # Create and start a connection pool with 4 connections
-#   pool = Pgtk::Pool.new(wire)
-#   pool.start!(4)
+#   pool = Pgtk::Pool.new(wire, max: 4)
+#   pool.start!
 #
 #   # Execute a simple query
 #   pool.exec('SELECT * FROM users')
@@ -51,11 +52,14 @@ class Pgtk::Pool
   # Constructor.
   #
   # @param [Pgtk::Wire] wire The wire
+  # @param [Integer] max Total amount of PostgreSQL connections in the pool
   # @param [Object] log The log
-  def initialize(wire, log: Loog::NULL)
+  def initialize(wire, max: 8, log: Loog::NULL)
     @wire = wire
+    @max = max
     @log = log
-    @pool = IterableQueue.new
+    @pool = IterableQueue.new(max)
+    @started = false
   end
 
   # Get the version of PostgreSQL server.
@@ -71,6 +75,7 @@ class Pgtk::Pool
   def dump
     [
       'Pgtk::Pool',
+      "  Pgtk version: #{Pgtk::VERSION}",
       "  PgSQL version: #{version}",
       "  #{@pool.size} connections:",
       @pool.map do |c|
@@ -120,13 +125,13 @@ class Pgtk::Pool
   # keep in mind that not all servers will allow you to have many connections
   # open at the same time. For example, Heroku free PostgreSQL database
   # allows only one connection open.
-  #
-  # @param [Integer] max Total amount of PostgreSQL connections in the pool
-  def start!(max = 8)
-    max.times do
+  def start!
+    return if @started
+    @max.times do
       @pool << @wire.connection
     end
-    @log.debug("PostgreSQL pool started with #{max} connections")
+    @started = true
+    @log.debug("PostgreSQL pool started with #{@max} connections")
   end
 
   # Make a query and return the result as an array of hashes. For example,
@@ -219,24 +224,43 @@ class Pgtk::Pool
   #
   # This class is used internally by Pool to store database connections
   # and provide the ability to iterate over them for inspection purposes.
+  #
+  # The queue is bounded by size. When an item is taken out, it remains in
+  # the internal array but is marked as "taken". When returned, it's placed
+  # back in its original slot and marked as available.
   class IterableQueue
-    def initialize
+    def initialize(size)
+      @size = size
       @items = []
+      @taken = []
       @mutex = Mutex.new
       @condition = ConditionVariable.new
     end
 
     def <<(item)
       @mutex.synchronize do
-        @items << item
+        if @items.size < @size
+          @items << item
+          @taken << false
+        else
+          index = @items.index(item)
+          if index.nil?
+            index = @taken.index(true)
+            raise 'No taken slot found' if index.nil?
+            @items[index] = item
+          end
+          @taken[index] = false
+        end
         @condition.signal
       end
     end
 
     def pop
       @mutex.synchronize do
-        @condition.wait(@mutex) while @items.empty?
-        @items.shift
+        @condition.wait(@mutex) while @taken.all? || @items.empty?
+        index = @taken.index(false)
+        @taken[index] = true
+        @items[index]
       end
     end
 

data/lib/pgtk/retry.rb

@@ -16,8 +16,8 @@ require_relative '../pgtk'
 # Basic usage:
 #
 #   # Create and configure a regular pool
-#   pool = Pgtk::Pool.new(wire)
-#   pool.start!(4)
+#   pool = Pgtk::Pool.new(wire, max: 4)
+#   pool.start!
 #
 #   # Wrap the pool in a retry decorator with 3 attempts
 #   retry_pool = Pgtk::Retry.new(pool, attempts: 3)
@@ -58,8 +58,8 @@ class Pgtk::Retry
   end
 
   # Start a new connection pool with the given arguments.
-  def start!(*)
-    @pool.start!(*)
+  def start!
+    @pool.start!
   end
 
   # Get the version of PostgreSQL server.

data/lib/pgtk/spy.rb

@@ -19,8 +19,8 @@ require_relative 'wire'
 # Basic usage:
 #
 #   # Create and configure a regular pool
-#   pool = Pgtk::Pool.new(wire)
-#   pool.start!(4)
+#   pool = Pgtk::Pool.new(wire, max: 4)
+#   pool.start!
 #
 #   # Wrap the pool in a spy that tracks all executed queries
 #   queries = []
@@ -57,8 +57,8 @@ class Pgtk::Spy
   end
 
   # Start a new connection pool with the given arguments.
-  def start!(*)
-    @pool.start!(*)
+  def start!
+    @pool.start!
   end
 
   # Get the version of PostgreSQL server.

data/lib/pgtk/stash.rb

@@ -6,6 +6,7 @@
 require 'concurrent-ruby'
 require 'joined'
 require 'loog'
+require 'tago'
 require_relative '../pgtk'
 
 # Database query cache implementation.
@@ -39,6 +40,8 @@ class Pgtk::Stash
   # @option [Hash] queries Internal cache data (default: {})
   # @option [Hash] tables Internal cache data (default: {})
   # @param [Integer] refill_interval Interval in seconds for recalculate stale queries
+  # @param [Integer] cap How many queries to keep in cache (if more, oldest ones are deleted)
+  # @param [Integer] cap_interval Interval in seconds for cap the cache (remove old queries)
   # @param [Integer] max_queue_length Number of refilling tasks in the queue
   # @param [Integer] threads Number of threads in tpool
   # @param [Loog] loog Logger for debugging (default: null logger)
@@ -48,6 +51,8 @@ class Pgtk::Stash
     refill_interval: 16,
     max_queue_length: 128,
     threads: 4,
+    cap: 10_000,
+    cap_interval: 60,
     loog: Loog::NULL,
     entrance: Concurrent::ReentrantReadWriteLock.new,
     launched: Concurrent::AtomicBoolean.new(false)
@@ -59,14 +64,16 @@ class Pgtk::Stash
     @refill_interval = refill_interval
     @max_queue_length = max_queue_length
     @threads = threads
+    @cap = cap
+    @cap_interval = cap_interval
     @loog = loog
     @tpool = Concurrent::FixedThreadPool.new(@threads)
   end
 
   # Start a new connection pool with the given arguments.
-  def start!(*)
+  def start!
     launch!
-    @pool.start!(*)
+    @pool.start!
   end
 
   # Get the PostgreSQL server version.
@@ -78,13 +85,14 @@ class Pgtk::Stash
   # Convert internal state into text.
   def dump
     qq =
-      @stash[:queries].map do |k, v|
-        [
-          k.dup, # the query
-          v.values.count, # how many keys?
-          v.values.sum { |vv| vv[:popularity] }, # total popularity of all keys
-          v.values.count { |vv| vv[:stale] } # how many stale keys?
-        ]
+      @stash[:queries].map do |q, kk|
+        {
+          q: q.dup, # the query
+          c: kk.values.count, # how many keys?
+          p: kk.values.sum { |vv| vv[:popularity] }, # total popularity of all keys
+          s: kk.values.count { |vv| vv[:stale] }, # how many stale keys?
+          u: kk.values.map { |vv| vv[:used] }.max || Time.now # when was it used
+        }
       end
     [
       @pool.dump,
@@ -93,10 +101,14 @@ class Pgtk::Stash
       "  #{'not ' if @launched.false?}launched",
       "  #{@tpool.queue_length} task(s) in the thread pool",
       "  #{@stash[:tables].count} table(s) in cache",
-      "  #{qq.sum { |a| a[3] }} stale quer(ies) in cache:",
-      qq.select { |a| a[3].positive? }.sort_by { -_1[2] }.take(16).map { |a| "    #{a[1]}/#{a[2]}p/#{a[3]}s: #{a[0]}" },
-      "  #{qq.count { |a| a[3].zero? }} other quer(ies) in cache:",
-      qq.select { |a| a[3].zero? }.sort_by { -_1[2] }.take(8).map { |a| "    #{a[1]}/#{a[2]}p/#{a[3]}s: #{a[0]}" }
+      "  #{qq.sum { |a| a[:s] }} stale quer(ies) in cache:",
+      qq.select { |a| a[:s].positive? }.sort_by { -_1[:p] }.take(8).map do |a|
+        "    #{a[:c]}/#{a[:p]}p/#{a[:s]}s/#{a[:u].ago}: #{a[:q]}"
+      end,
+      "  #{qq.count { |a| a[:s].zero? }} other quer(ies) in cache:",
+      qq.select { |a| a[:s].zero? }.sort_by { -_1[:p] }.take(16).map do |a|
+        "    #{a[:c]}/#{a[:p]}p/#{a[:s]}s/#{a[:u].ago}: #{a[:q]}"
+      end
     ].join("\n")
   end
 
@@ -136,7 +148,7 @@ class Pgtk::Stash
               @stash[:tables][t].append(pure).uniq!
             end
             @stash[:queries][pure] ||= {}
-            @stash[:queries][pure][key] = { ret:, params:, result: }
+            @stash[:queries][pure][key] = { ret:, params:, result:, used: Time.now }
           end
         end
       end
@@ -144,6 +156,7 @@ class Pgtk::Stash
         @entrance.with_write_lock do
           @stash[:queries][pure][key][:popularity] ||= 0
           @stash[:queries][pure][key][:popularity] += 1
+          @stash[:queries][pure][key][:used] = Time.now
         end
       end
     end
@@ -175,11 +188,16 @@ class Pgtk::Stash
 
   def launch!
     raise 'Cannot launch multiple times on same cache data' unless @launched.make_true
-    Concurrent::TimerTask.execute(execution_interval: 60 * 60, executor: @tpool) do
-      @entrance.with_write_lock do
-        @stash[:queries].each_key do |q|
-          @stash[:queries][q].each_key do |k|
-            @stash[:queries][q][k][:popularity] = 0
+    Concurrent::TimerTask.execute(execution_interval: @cap_interval, executor: @tpool) do
+      loop do
+        s = @stash[:queries].values.sum { |kk| kk.values.size }
+        break if s <= @cap
+        @entrance.with_write_lock do
+          @stash[:queries].each_key do |q|
+            m = @stash[:queries][q].values.map { |h| h[:used] }.min
+            next unless m
+            @stash[:queries][q].delete_if { |_, h| h[:used] == m }
+            @stash[:queries].delete_if { |_, kk| kk.empty? }
           end
         end
       end

data/lib/pgtk/version.rb

@@ -11,5 +11,5 @@ require_relative '../pgtk'
 # License:: MIT
 module Pgtk
   # Current version of the library.
-  VERSION = '0.22.1'
+  VERSION = '0.24.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgtk
 version: !ruby/object:Gem::Version
-  version: 0.22.1
+  version: 0.24.0
 platform: ruby
 authors:
 - Yegor Bugayenko