activerecord-enhancedsqlite3-adapter 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24efb62453474faa0b37357d98f5b0f8ba13fa98d7e633b5d3a7de5e16d875f6
-  data.tar.gz: 0de2b8cae842c4262d81932c577df7731e570f347f0a22a2151ef9671bd8648d
+  metadata.gz: 6bd667df1779690780802a3f91133b9a6b8155f9f3c040393b5a722a299ce370
+  data.tar.gz: ed4008074efd597d2bc79ab2a4b3aea8e7ccccb37fd8158892843d20918b0015
 SHA512:
-  metadata.gz: a55c35f42fef85b0373df52f06d1b50b66c4191382844ee6a9836049af03386085bfe9616de8bfc259fcfd6325d5a84f4cc8ca396d721ff87fdcfc49c80dae17
-  data.tar.gz: b50f2484872aa18a0681c31abfd4d6a56e044ff9610cb288c73611cb0f56ed8b8242a771b9c4e95a07119ec094aef5bcf6c85f1bc363fd08e59d45a57b162902
+  metadata.gz: 3252cf16cc3619a857488b450542ef6b9402b666bdb1a5458d3615d880c073226cbf8d7472032b74bcfa205196f4febf0aaa8bc1fa2e6e00f00dc0d66e975d97
+  data.tar.gz: ddbafb131266c8edcf5e66303083ab964e8ac9248eaa12eeea4c377501f00e820b54edf406280eb832f0996120e666434e256e308a108d6df945bb21168f8416

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [0.5.0] - 2023-12-24
+
+- Load extensions installed via project-scoped `sqlpkg`
+
+## [0.4.0] - 2023-12-10
+
+- Ensure transactions are IMMEDIATE and not DEFERRED
+- Ensure that our `busy_handler` is the very first configuration to be set on a connection
+- Simplify and speed up our `busy_handler` implementation
+
 ## [0.3.0] - 2023-12-06
 
 - Added a more performant implementation of the the `timeout` mechanism

data/lib/enhanced_sqlite3/adapter.rb

@@ -10,6 +10,27 @@ require "enhanced_sqlite3/supports_deferrable_constraints"
 
 module EnhancedSQLite3
   module Adapter
+    # Setup the Rails SQLite3 adapter instance.
+    #
+    # extends  https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L90
+    def initialize(...)
+      super
+      # Ensure that all connections default to immediate transaction mode.
+      # This is necessary to prevent SQLite from deadlocking when concurrent processes open write transactions.
+      # By default, SQLite opens transactions in deferred mode, which means that a transactions acquire
+      # a shared lock on the database, but will attempt to upgrade that lock to an exclusive lock if/when
+      # a write is attempted. Because SQLite is in the middle of a transaction, it cannot retry the transaction
+      # if a BUSY exception is raised, and so it will immediately raise a SQLITE_BUSY exception without calling
+      # the `busy_handler`. Because Rails only wraps writes in transactions, this means that all transactions
+      # will attempt to acquire an exclusive lock on the database. Thus, under any concurrent load, you are very
+      # likely to encounter a SQLITE_BUSY exception.
+      # By setting the default transaction mode to immediate, SQLite will instead attempt to acquire
+      # an exclusive lock as soon as the transaction is opened. If the lock cannot be acquired, it will
+      # immediately call the `busy_handler` to retry the transaction. This allows concurrent processes to
+      # coordinate and linearize their transactions, avoiding deadlocks.
+      @connection_parameters.merge!(default_transaction_mode: :immediate)
+    end
+
     # Perform any necessary initialization upon the newly-established
     # @raw_connection -- this is the place to modify the adapter's
     # connection settings, run queries to configure any application-global
@@ -18,13 +39,12 @@ module EnhancedSQLite3
     # Implementations may assume this method will only be called while
     # holding @lock (or from #initialize).
     #
-    # extends https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L691
+    # overrides https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L691
     def configure_connection
-      super
-
       configure_busy_handler_timeout
+      check_version
       configure_pragmas
-      configure_extensions
+      load_extensions
 
       EnhancedSQLite3::SupportsVirtualColumns.apply!
       EnhancedSQLite3::SupportsDeferrableConstraints.apply!
@@ -36,32 +56,67 @@ module EnhancedSQLite3
       return unless @config.key?(:timeout)
 
       timeout = self.class.type_cast_config_to_integer(@config[:timeout])
+      timeout_seconds = timeout.fdiv(1000)
+      retry_interval = 6e-5 # 60 microseconds
+
       @raw_connection.busy_handler do |count|
         timed_out = false
-        # capture the start time of this blocked write
-        @start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) if count == 0
         # keep track of elapsed time every 100 iterations (to lower load)
-        if count % 100 == 0
-          @elapsed_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_time
+        if (count % 100).zero?
           # fail if we exceed the timeout value (captured from the timeout config option, converted to seconds)
-          timed_out = @elapsed_time > timeout
+          timed_out = (count * retry_interval) > timeout_seconds
         end
         if timed_out
           false # this will cause the BusyException to be raised
         else
-          sleep 0.001 # sleep 1 millisecond (or whatever)
+          sleep(retry_interval)
+          true
         end
       end
     end
 
     def configure_pragmas
-      @config.fetch(:pragmas, []).each do |key, value|
+      defaults = {
+        # Enforce foreign key constraints
+        # https://www.sqlite.org/pragma.html#pragma_foreign_keys
+        # https://www.sqlite.org/foreignkeys.html
+        "foreign_keys" => "ON",
+        # Impose a limit on the WAL file to prevent unlimited growth
+        # https://www.sqlite.org/pragma.html#pragma_journal_size_limit
+        "journal_size_limit" => 64.megabytes,
+        # Set the local connection cache to 2000 pages
+        # https://www.sqlite.org/pragma.html#pragma_cache_size
+        "cache_size" => 2000
+      }
+      unless @memory_database
+        defaults.merge!(
+          # Journal mode WAL allows for greater concurrency (many readers + one writer)
+          # https://www.sqlite.org/pragma.html#pragma_journal_mode
+          "journal_mode" => "WAL",
+          # Set more relaxed level of database durability
+          # 2 = "FULL" (sync on every write), 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE"
+          # https://www.sqlite.org/pragma.html#pragma_synchronous
+          "synchronous" => "NORMAL",
+          # Set the global memory map so all processes can share some data
+          # https://www.sqlite.org/pragma.html#pragma_mmap_size
+          # https://www.sqlite.org/mmap.html
+          "mmap_size" => 128.megabytes
+        )
+      end
+      pragmas = defaults.merge(@config.fetch(:pragmas, {}))
+
+      pragmas.each do |key, value|
         execute("PRAGMA #{key} = #{value}", "SCHEMA")
       end
     end
 
-    def configure_extensions
+    def load_extensions
       @raw_connection.enable_load_extension(true)
+      # first, load any extensions installed via `sqlpkg`
+      Dir.glob(".sqlpkg/**/*.{dll,so,dylib}") do |extension_path|
+        @raw_connection.load_extension(extension_path)
+      end
+      # then, load any extensions specified in the `database.yml`
       @config.fetch(:extensions, []).each do |extension_name|
         require extension_name
         extension_classname = extension_name.camelize

data/lib/enhanced_sqlite3/version.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 module EnhancedSQLite3
-  VERSION = '0.3.0'
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-enhancedsqlite3-adapter
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Stephen Margheim
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-12-06 00:00:00.000000000 Z
+date: 2023-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -137,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.5.1
 signing_key:
 specification_version: 4
 summary: ActiveRecord adapter for SQLite that enhances the default.