activerecord-enhancedsqlite3-adapter 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24efb62453474faa0b37357d98f5b0f8ba13fa98d7e633b5d3a7de5e16d875f6
-  data.tar.gz: 0de2b8cae842c4262d81932c577df7731e570f347f0a22a2151ef9671bd8648d
+  metadata.gz: fd134c96488a861c3fdc36c32396547a31ea596c11243aff923995acff8ae4b5
+  data.tar.gz: d2029b2162c66df5689e65cb4e281f73d21049adafaecf9bbf78490570a10321
 SHA512:
-  metadata.gz: a55c35f42fef85b0373df52f06d1b50b66c4191382844ee6a9836049af03386085bfe9616de8bfc259fcfd6325d5a84f4cc8ca396d721ff87fdcfc49c80dae17
-  data.tar.gz: b50f2484872aa18a0681c31abfd4d6a56e044ff9610cb288c73611cb0f56ed8b8242a771b9c4e95a07119ec094aef5bcf6c85f1bc363fd08e59d45a57b162902
+  metadata.gz: 6803168a5f067839e6ea7416f2f067bbc044202d14fd21a9536d068bebe35d7d89a481cbfcf5e230d4063705a7b5241aeb76d8f92846e2ecf5cdc2afa8b805f0
+  data.tar.gz: 2845ea0983129dd0e6e66b66e8f10fca3538192dcdaec62ec55f497ebb8f82f52bdc2352eb1fdf85b859bbd7436e86b57d15da4ce1135c00e23f8996db5cfc59

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## [Unreleased]
 
+## [0.4.0] - 2023-12-10
+
+- Ensure transactions are IMMEDIATE and not DEFERRED
+- Ensure that our `busy_handler` is the very first configuration to be set on a connection
+- Simplify and speed up our `busy_handler` implementation
+
 ## [0.3.0] - 2023-12-06
 
 - Added a more performant implementation of the the `timeout` mechanism

data/lib/enhanced_sqlite3/adapter.rb

@@ -10,6 +10,27 @@ require "enhanced_sqlite3/supports_deferrable_constraints"
 
 module EnhancedSQLite3
   module Adapter
+    # Setup the Rails SQLite3 adapter instance.
+    #
+    # extends  https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L90
+    def initialize(...)
+      super
+      # Ensure that all connections default to immediate transaction mode.
+      # This is necessary to prevent SQLite from deadlocking when concurrent processes open write transactions.
+      # By default, SQLite opens transactions in deferred mode, which means that a transactions acquire
+      # a shared lock on the database, but will attempt to upgrade that lock to an exclusive lock if/when
+      # a write is attempted. Because SQLite is in the middle of a transaction, it cannot retry the transaction
+      # if a BUSY exception is raised, and so it will immediately raise a SQLITE_BUSY exception without calling
+      # the `busy_handler`. Because Rails only wraps writes in transactions, this means that all transactions
+      # will attempt to acquire an exclusive lock on the database. Thus, under any concurrent load, you are very
+      # likely to encounter a SQLITE_BUSY exception.
+      # By setting the default transaction mode to immediate, SQLite will instead attempt to acquire
+      # an exclusive lock as soon as the transaction is opened. If the lock cannot be acquired, it will
+      # immediately call the `busy_handler` to retry the transaction. This allows concurrent processes to
+      # coordinate and linearize their transactions, avoiding deadlocks.
+      @connection_parameters.merge!(default_transaction_mode: :immediate)
+    end
+
     # Perform any necessary initialization upon the newly-established
     # @raw_connection -- this is the place to modify the adapter's
     # connection settings, run queries to configure any application-global
@@ -18,11 +39,10 @@ module EnhancedSQLite3
     # Implementations may assume this method will only be called while
     # holding @lock (or from #initialize).
     #
-    # extends https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L691
+    # overrides https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L691
     def configure_connection
-      super
-
       configure_busy_handler_timeout
+      check_version
       configure_pragmas
       configure_extensions
 
@@ -36,26 +56,56 @@ module EnhancedSQLite3
       return unless @config.key?(:timeout)
 
       timeout = self.class.type_cast_config_to_integer(@config[:timeout])
+      timeout_seconds = timeout.fdiv(1000)
+      retry_interval = 6e-5 # 60 microseconds
+
       @raw_connection.busy_handler do |count|
         timed_out = false
-        # capture the start time of this blocked write
-        @start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) if count == 0
         # keep track of elapsed time every 100 iterations (to lower load)
-        if count % 100 == 0
-          @elapsed_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_time
+        if (count % 100).zero?
           # fail if we exceed the timeout value (captured from the timeout config option, converted to seconds)
-          timed_out = @elapsed_time > timeout
+          timed_out = (count * retry_interval) > timeout_seconds
         end
         if timed_out
           false # this will cause the BusyException to be raised
         else
-          sleep 0.001 # sleep 1 millisecond (or whatever)
+          sleep(retry_interval)
+          true
         end
       end
     end
 
     def configure_pragmas
-      @config.fetch(:pragmas, []).each do |key, value|
+      defaults = {
+        # Enforce foreign key constraints
+        # https://www.sqlite.org/pragma.html#pragma_foreign_keys
+        # https://www.sqlite.org/foreignkeys.html
+        "foreign_keys" => "ON",
+        # Impose a limit on the WAL file to prevent unlimited growth
+        # https://www.sqlite.org/pragma.html#pragma_journal_size_limit
+        "journal_size_limit" => 64.megabytes,
+        # Set the local connection cache to 2000 pages
+        # https://www.sqlite.org/pragma.html#pragma_cache_size
+        "cache_size" => 2000
+      }
+      unless @memory_database
+        defaults.merge!(
+          # Journal mode WAL allows for greater concurrency (many readers + one writer)
+          # https://www.sqlite.org/pragma.html#pragma_journal_mode
+          "journal_mode" => "WAL",
+          # Set more relaxed level of database durability
+          # 2 = "FULL" (sync on every write), 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE"
+          # https://www.sqlite.org/pragma.html#pragma_synchronous
+          "synchronous" => "NORMAL",
+          # Set the global memory map so all processes can share some data
+          # https://www.sqlite.org/pragma.html#pragma_mmap_size
+          # https://www.sqlite.org/mmap.html
+          "mmap_size" => 128.megabytes
+        )
+      end
+      pragmas = defaults.merge(@config.fetch(:pragmas, {}))
+
+      pragmas.each do |key, value|
         execute("PRAGMA #{key} = #{value}", "SCHEMA")
       end
     end

data/lib/enhanced_sqlite3/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module EnhancedSQLite3
-  VERSION = '0.3.0'
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-enhancedsqlite3-adapter
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Stephen Margheim
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-12-06 00:00:00.000000000 Z
+date: 2023-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord