fbe 0.19.2 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2d7532cc824357975d405225601f1f0873721e96b5328a9cd4e1d57191008a9
-  data.tar.gz: 1c64cade131c3f2751c0dffaaa9bd157cdc4ff41a70b0acf5efe8a8f2788e08b
+  metadata.gz: 8c33c559724c991d4aabc2ad9c097170e53de0002e362dcfadfe956d77eed7ec
+  data.tar.gz: 787dbc8ce12ebacbd0b96cd8174ced19aa11665f29d3d410fb93c397dbb80d87
 SHA512:
-  metadata.gz: 742ec0b8b9970a86549a8292ad5390cb5dcdb4addbc4c45590f3ff10526a67d4038c65cc9534d818859558a0915efeeef30d165816cd24a9a12cda2ec83cc87e
-  data.tar.gz: 70eb53d6eb30817c735ef3ab2ea62e050106b616a8384a647106c0ef4b40e861deb62d6a3dfc43a32471157144c9225425e092e4a4df614e91679444c5219f64
+  metadata.gz: ca43bc96f5c8f426a72fb3ab722684d36b67613526b9d2e1be175abae03581c4fc0318314e92628c9064d2c7ba8637814ed950e349ba86b64caedc768619be8f
+  data.tar.gz: 81d737f13af364095e871df44000f6293d1c062929d7f859cd3836d23c3cf602a1362a9653fd739f1726c4d1004e0bad1806e3cb3fde23dc4aef081e39dfbd6a

data/Gemfile

@@ -8,6 +8,7 @@ gemspec
 
 gem 'minitest', '~>5.25', require: false
 gem 'minitest-reporters', '~>1.7', require: false
+gem 'minitest-stub-const', '~>0.6', require: false
 gem 'os', '~>1.1', require: false
 gem 'qbash', '~>0.4', require: false
 gem 'rake', '~>13.3', require: false

data/Gemfile.lock

@@ -143,6 +143,7 @@ GEM
       builder
       minitest (>= 5.0)
       ruby-progressbar
+    minitest-stub-const (0.6)
     moments (0.3.0)
     multipart-post (2.4.1)
     net-http (0.6.0)
@@ -179,7 +180,7 @@ GEM
     regexp_parser (2.10.0)
     retries (0.0.5)
     rexml (3.4.1)
-    rubocop (1.76.0)
+    rubocop (1.76.1)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -190,7 +191,7 @@ GEM
       rubocop-ast (>= 1.45.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.45.0)
+    rubocop-ast (1.45.1)
       parser (>= 3.3.7.2)
       prism (~> 1.4)
     rubocop-minitest (0.38.1)
@@ -258,6 +259,7 @@ DEPENDENCIES
   fbe!
   minitest (~> 5.25)
   minitest-reporters (~> 1.7)
+  minitest-stub-const (~> 0.6)
   os (~> 1.1)
   qbash (~> 0.4)
   rake (~> 13.3)

data/lib/fbe/middleware/sqlite_store.rb

@@ -3,6 +3,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2024-2025 Zerocracy
 # SPDX-License-Identifier: MIT
 
+require 'time'
 require 'json'
 require 'sqlite3'
 require_relative '../../fbe'
@@ -16,18 +17,20 @@ require_relative '../../fbe/middleware'
 class Fbe::Middleware::SqliteStore
   attr_reader :path
 
-  def initialize(path)
+  def initialize(path, version)
     raise ArgumentError, 'Database path cannot be nil or empty' if path.nil? || path.empty?
     dir = File.dirname(path)
     raise ArgumentError, "Directory #{dir} does not exist" unless File.directory?(dir)
+    raise ArgumentError, 'Version cannot be nil or empty' if version.nil? || version.empty?
     @path = File.absolute_path(path)
-    open
-    prepare
-    at_exit { close }
+    @version = version
   end
 
   def read(key)
-    value = perform { _1.execute('SELECT value FROM cache WHERE key = ? LIMIT 1', [key]) }.dig(0, 0)
+    value = perform do |t|
+      t.execute('UPDATE cache SET touched_at = ?2 WHERE key = ?1;', [key, Time.now.utc.iso8601])
+      t.execute('SELECT value FROM cache WHERE key = ? LIMIT 1;', [key])
+    end.dig(0, 0)
     JSON.parse(value) if value
   end
 
@@ -37,40 +40,27 @@ class Fbe::Middleware::SqliteStore
   end
 
   def write(key, value)
+    return if value.is_a?(Array) && value.any? do |vv|
+      req = JSON.parse(vv[0])
+      req['url'].include?('?') || req['method'] != 'get'
+    end
     value = JSON.dump(value)
-    perform do |tdb|
-      tdb.execute(<<~SQL, [key, value])
-        INSERT INTO cache(key, value) VALUES(?1, ?2)
-        ON CONFLICT(key) DO UPDATE SET value = ?2
+    return if value.bytesize > 10_000
+    perform do |t|
+      t.execute(<<~SQL, [key, value, Time.now.utc.iso8601])
+        INSERT INTO cache(key, value, touched_at) VALUES(?1, ?2, ?3)
+        ON CONFLICT(key) DO UPDATE SET value = ?2, touched_at = ?3
       SQL
     end
     nil
   end
 
-  def open
-    return if @db
-    @db = SQLite3::Database.new(@path)
-  end
-
-  def prepare
-    perform do |tdb|
-      tdb.execute 'CREATE TABLE IF NOT EXISTS cache(key TEXT UNIQUE NOT NULL, value TEXT);'
-      tdb.execute 'CREATE INDEX IF NOT EXISTS key_idx ON cache(key);'
-    end
-  end
-
-  def close
-    return if !@db || @db.closed?
-    @db.close
-    @db = nil
-  end
-
   def clear
-    perform { _1.execute 'DELETE FROM cache;' }
-  end
-
-  def drop
-    perform { _1.execute 'DROP TABLE IF EXISTS cache;' }
+    perform do |t|
+      t.execute 'DELETE FROM cache;'
+      t.execute "UPDATE meta SET value = ? WHERE key = 'version';", [@version]
+    end
+    @db.execute 'VACUUM;'
   end
 
   def all
@@ -80,6 +70,43 @@ class Fbe::Middleware::SqliteStore
   private
 
   def perform(&)
+    @db ||=
+      SQLite3::Database.new(@path).tap do |d|
+        d.transaction do |t|
+          t.execute <<~SQL
+            CREATE TABLE IF NOT EXISTS cache(
+              key TEXT UNIQUE NOT NULL, value TEXT, touched_at TEXT NOT NULL
+            );
+          SQL
+          t.execute 'CREATE INDEX IF NOT EXISTS cache_key_idx ON cache(key);'
+          t.execute 'CREATE INDEX IF NOT EXISTS cache_touched_at_idx ON cache(touched_at);'
+          t.execute 'CREATE TABLE IF NOT EXISTS meta(key TEXT UNIQUE NOT NULL, value TEXT);'
+          t.execute 'CREATE INDEX IF NOT EXISTS meta_key_idx ON meta(key);'
+          t.execute "INSERT INTO meta(key, value) VALUES('version', ?) ON CONFLICT(key) DO NOTHING;", [@version]
+        end
+        if d.execute("SELECT value FROM meta WHERE key = 'version' LIMIT 1;").dig(0, 0) != @version
+          d.transaction do |t|
+            t.execute 'DELETE FROM cache;'
+            t.execute "UPDATE meta SET value = ? WHERE key = 'version';", [@version]
+          end
+          d.execute 'VACUUM;'
+        end
+        if File.size(@path) > 10 * 1024 * 1024
+          while d.execute(<<~SQL).dig(0, 0) > 10 * 1024 * 1024
+            SELECT (page_count - freelist_count) * page_size AS size
+            FROM pragma_page_count(), pragma_freelist_count(), pragma_page_size();
+          SQL
+            d.transaction do |t|
+              t.execute <<~SQL
+                DELETE FROM cache
+                WHERE key IN (SELECT key FROM cache ORDER BY touched_at LIMIT 50)
+              SQL
+            end
+          end
+          d.execute 'VACUUM;'
+        end
+        at_exit { @db&.close }
+      end
     @db.transaction(&)
   end
 end

data/lib/fbe/octo.rb

@@ -76,16 +76,24 @@ def Fbe.octo(options: $options, global: $global, loog: $loog)
               methods: [:get],
               backoff_factor: 2
             )
-            serializer = Marshal
             if options.sqlite_cache
-              store = Fbe::Middleware::SqliteStore.new(options.sqlite_cache)
-              serializer = JSON
-              loog.info("Using HTTP cache in SQLite file: #{store.path}")
+              store = Fbe::Middleware::SqliteStore.new(options.sqlite_cache, Fbe::VERSION)
+              loog.info(
+                "Using HTTP cache in SQLite file: #{store.path} (" \
+                "#{File.exist?(store.path) ? "#{File.size(store.path)} bytes" : 'file is absent'}" \
+                ')'
+              )
+              builder.use(
+                Faraday::HttpCache,
+                store:, serializer: JSON, shared_cache: false, logger: Loog::NULL
+              )
+            else
+              loog.info("No HTTP cache in SQLite file, because 'sqlite_cache' option is not provided")
+              builder.use(
+                Faraday::HttpCache,
+                serializer: Marshal, shared_cache: false, logger: Loog::NULL
+              )
             end
-            builder.use(
-              Faraday::HttpCache,
-              store: store, serializer: serializer, shared_cache: false, logger: Loog::NULL
-            )
             builder.use(Octokit::Response::RaiseError)
             builder.use(Faraday::Response::Logger, loog, formatter: Fbe::Middleware::Formatter)
             builder.use(Fbe::Middleware::Trace, trace)

data/lib/fbe.rb

@@ -10,5 +10,5 @@
 # License:: MIT
 module Fbe
   # Current version of the gem (changed by +.rultor.yml+ on every release)
-  VERSION = '0.19.2' unless const_defined?(:VERSION)
+  VERSION = '0.20.0' unless const_defined?(:VERSION)
 end

data/test/fbe/middleware/test_sqlite_store.rb

@@ -3,6 +3,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2024-2025 Zerocracy
 # SPDX-License-Identifier: MIT
 
+require 'qbash'
 require_relative '../../test__helper'
 require_relative '../../../lib/fbe/middleware'
 require_relative '../../../lib/fbe/middleware/sqlite_store'
@@ -12,24 +13,180 @@ require_relative '../../../lib/fbe/middleware/sqlite_store'
 # Copyright:: Copyright (c) 2024-2025 Zerocracy
 # License:: MIT
 class SqliteStoreTest < Fbe::Test
-  def test_sqlite_store
-    Dir.mktmpdir do |dir|
-      store = Fbe::Middleware::SqliteStore.new(File.expand_path('test.db', dir))
-      assert_nil(store.read('my_key'))
-      assert_nil(store.delete('my_key'))
-      assert_nil(store.write('my_key', 'some value'))
-      assert_equal('some value', store.read('my_key'))
-      assert_nil(store.write('my_key', 'some value 2'))
-      assert_equal('some value 2', store.read('my_key'))
-      assert_nil(store.delete('my_key'))
-      assert_nil(store.read('my_key'))
+  def test_simple_caching_algorithm
+    with_tmpfile('x.db') do |f|
+      store = Fbe::Middleware::SqliteStore.new(f, '0.0.0')
+      k = 'some-key'
+      assert_nil(store.read(k))
+      assert_nil(store.delete(k))
+      v1 = 'first value to save'
+      assert_nil(store.write(k, v1))
+      assert_equal(v1, store.read(k))
+      v2 = 'another value to save'
+      assert_nil(store.write(k, v2))
+      assert_equal(v2, store.read(k))
+      assert_nil(store.delete(k))
+      assert_nil(store.read(k))
+      assert_path_exists(f)
     end
   end
 
-  def test_sqlite_store_empty_all
-    Dir.mktmpdir do |dir|
-      store = Fbe::Middleware::SqliteStore.new(File.expand_path('test.db', dir))
+  def test_returns_empty_list
+    with_tmpfile('b.db') do |f|
+      store = Fbe::Middleware::SqliteStore.new(f, '0.0.0')
+      assert_empty(store.all)
+    end
+  end
+
+  def test_clear_all_keys
+    with_tmpfile('a.db') do |f|
+      store = Fbe::Middleware::SqliteStore.new(f, '0.0.0')
+      k = 'a key'
+      store.write(k, 'some value')
+      store.clear
+      assert_empty(store.all)
+    end
+  end
+
+  def test_empty_all_if_not_written
+    with_tmpfile do |f|
+      store = Fbe::Middleware::SqliteStore.new(f, '0.0.0')
       assert_empty(store.all)
     end
   end
+
+  def test_wrong_db_path
+    assert_raises(ArgumentError) do
+      Fbe::Middleware::SqliteStore.new(nil, '0.0.0').read('my_key')
+    end
+    assert_raises(ArgumentError) do
+      Fbe::Middleware::SqliteStore.new('', '0.0.0').read('my_key')
+    end
+    assert_raises(ArgumentError) do
+      Fbe::Middleware::SqliteStore.new('/fakepath/fakefolder/test.db', '0.0.0').read('my_key')
+    end
+  end
+
+  def test_not_db_file
+    with_tmpfile do |f|
+      File.binwrite(f, Array.new(20) { rand(0..255) }.pack('C*'))
+      ex =
+        assert_raises(SQLite3::NotADatabaseException) do
+          Fbe::Middleware::SqliteStore.new(f, '0.0.0').read('my_key')
+        end
+      assert_match('file is not a database', ex.message)
+    end
+  end
+
+  def test_defer_db_close_callback
+    txt = <<~RUBY
+      require 'tempfile'
+      require 'sqlite3'
+      require 'fbe/middleware/sqlite_store'
+
+      SQLite3::Database.class_eval do
+        prepend(Module.new do
+          def close
+            super
+            puts 'closed sqlite after process exit'
+          end
+        end)
+      end
+
+      Tempfile.open('test.db') do |f|
+        Fbe::Middleware::SqliteStore.new(f.path, '0.0.0').then do |s|
+          s.write('my_key', 'my_value')
+          s.read('my_key')
+        end
+      end
+    RUBY
+    out =
+      qbash(
+        'bundle exec ruby ' \
+        "-I#{Shellwords.escape(File.expand_path('../../../lib', __dir__))} " \
+        "-e #{Shellwords.escape(txt)} 2>&1"
+      )
+    assert_match('closed sqlite after process exit', out)
+  end
+
+  def test_different_versions
+    with_tmpfile('d.db') do |f|
+      Fbe::Middleware::SqliteStore.new(f, '0.0.1').then do |store|
+        store.write('kkk1', 'some value')
+        store.write('kkk2', 'another value')
+      end
+      Fbe::Middleware::SqliteStore.new(f, '0.0.1').then do |store|
+        assert_equal('some value', store.read('kkk1'))
+        assert_equal('another value', store.read('kkk2'))
+      end
+      Fbe::Middleware::SqliteStore.new(f, '0.0.2').then do |store|
+        assert_nil(store.read('kkk1'))
+        assert_nil(store.read('kkk2'))
+      end
+    end
+  end
+
+  def test_initialize_wrong_version
+    with_tmpfile('e.db') do |f|
+      msg = 'Version cannot be nil or empty'
+      assert_raises(ArgumentError) { Fbe::Middleware::SqliteStore.new(f, nil) }.then do |ex|
+        assert_match(msg, ex.message)
+      end
+      assert_raises(ArgumentError) { Fbe::Middleware::SqliteStore.new(f, '') }.then do |ex|
+        assert_match(msg, ex.message)
+      end
+    end
+  end
+
+  def test_skip_write_if_value_more_then_10k_bytes
+    with_tmpfile('a.db') do |f|
+      Fbe::Middleware::SqliteStore.new(f, '0.0.1').then do |store|
+        store.write('a', 'a' * 9_997)
+        store.write('b', 'b' * 9_998)
+        store.write('c', 'c' * 9_999)
+        store.write('d', 'd' * 10_000)
+        assert_equal('a' * 9_997, store.read('a'))
+        assert_equal('b' * 9_998, store.read('b'))
+        assert_nil(store.read('c'))
+        assert_nil(store.read('d'))
+      end
+    end
+  end
+
+  def test_shrink_cache_if_more_then_10_mb
+    with_tmpfile('large.db') do |f|
+      Fbe::Middleware::SqliteStore.new(f, '0.0.1').then do |store|
+        key = 'aaa'
+        alpha = ('a'..'z').to_a
+        store.write('a', 'aa')
+        Time.stub(:now, (Time.now - (5 * 60 * 60)).round) do
+          store.write('b', 'bb')
+          store.write('c', 'cc')
+        end
+        assert_equal('cc', store.read('c'))
+        Time.stub(:now, rand((Time.now - (5 * 60 * 60))..Time.now).round) do
+          10_240.times do
+            value = alpha.sample * rand(1024..2048)
+            store.write(key, value)
+            key = key.next
+          end
+        end
+      end
+      assert_operator(File.size(f), :>, 10 * 1024 * 1024)
+      Fbe::Middleware::SqliteStore.new(f, '0.0.1').then do |store|
+        assert_equal('aa', store.read('a'))
+        assert_nil(store.read('b'))
+        assert_equal('cc', store.read('c'))
+        assert_operator(File.size(f), :<=, 10 * 1024 * 1024)
+      end
+    end
+  end
+
+  private
+
+  def with_tmpfile(name = 'test.db', &)
+    Dir.mktmpdir do |dir|
+      yield File.expand_path(name, dir)
+    end
+  end
 end

data/test/fbe/test_octo.rb

@@ -256,7 +256,10 @@ class TestOcto < Fbe::Test
   end
 
   def test_reads_quota
-    WebMock.enable_net_connect!
+    WebMock.disable_net_connect!
+    stub_request(:get, 'https://api.github.com/rate_limit').to_return(
+      { body: '{}', headers: { 'X-RateLimit-Remaining' => '222' } }
+    )
     o = Fbe.octo(loog: Loog::VERBOSE, global: {}, options: Judges::Options.new({ 'github_api_pause' => 0.01 }))
     refute_nil(o.off_quota?)
   end
@@ -365,7 +368,7 @@ class TestOcto < Fbe::Test
     assert_includes second_output, 'GitHub API trace is empty'
   end
 
-  def test_sqlite_store
+  def test_works_via_sqlite_store
     WebMock.disable_net_connect!
     Dir.mktmpdir do |dir|
       global = {}
@@ -402,4 +405,45 @@ class TestOcto < Fbe::Test
       assert_path_exists(file)
     end
   end
+
+  def test_sqlite_store_for_use_in_different_versions
+    WebMock.disable_net_connect!
+    Dir.mktmpdir do |dir|
+      global = {}
+      stub =
+        stub_request(:get, 'https://api.github.com/user/42')
+          .to_return(
+            status: 200,
+            body: { login: 'user1' }.to_json,
+            headers: {
+              'Content-Type' => 'application/json',
+              'Cache-Control' => 'public, max-age=60, s-maxage=60',
+              'Etag' => 'W/"2ff9dd4c3153f006830b2b8b721f6a4bb400a1eb81a2e1fa0a3b846ad349b9ec"',
+              'Last-Modified' => 'Wed, 01 May 2025 20:00:00 GMT'
+            }
+          )
+      sqlite_cache = File.expand_path('test.db', dir)
+      Fbe.stub_const(:VERSION, '0.0.1') do
+        o = Fbe.octo(loog: Loog::NULL, global:, options: Judges::Options.new({ 'sqlite_cache' => sqlite_cache }))
+        assert_equal('user1', o.user_name_by_id(42))
+      end
+      WebMock.remove_request_stub(stub)
+      stub_request(:get, 'https://api.github.com/user/42')
+        .to_return(
+          status: 200,
+          body: { login: 'user2' }.to_json,
+          headers: {
+            'Content-Type' => 'application/json',
+            'Cache-Control' => 'public, max-age=60, s-maxage=60',
+            'Etag' => 'W/"2ff9dd4c3153f006830b2b8b721f6a4bb400a1eb81a2e1fa0a3b846ad349b9ec"',
+            'Last-Modified' => 'Wed, 01 May 2025 20:00:00 GMT'
+          }
+        )
+      global = {}
+      Fbe.stub_const(:VERSION, '0.0.2') do
+        o = Fbe.octo(loog: Loog::NULL, global:, options: Judges::Options.new({ 'sqlite_cache' => sqlite_cache }))
+        assert_equal('user2', o.user_name_by_id(42))
+      end
+    end
+  end
 end

data/test/test__helper.rb

@@ -31,6 +31,7 @@ end
 require 'minitest/reporters'
 Minitest::Reporters.use! [Minitest::Reporters::SpecReporter.new]
 
+require 'minitest/stub_const'
 require 'minitest/autorun'
 require 'webmock/minitest'
 require_relative '../lib/fbe'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fbe
 version: !ruby/object:Gem::Version
-  version: 0.19.2
+  version: 0.20.0
 platform: ruby
 authors:
 - Yegor Bugayenko