sqlite_shards 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 70840b03f18bb37414203a8218c534e498a0b9a249663d1b010b56c88f6948d8
+  data.tar.gz: d3c83ebabc64f00d8ec71bd24b007618628f3e18b33bafd91a3ea5845cbe432a
+SHA512:
+  metadata.gz: 667dfce81ea40beeac34d1a31f96c1cec3b965c441bd05843649db65a46a23e916ded2711e444d37e4bafde6f0a4a4b3a30c4828ca232acdc713f5459ef51fba
+  data.tar.gz: 7510233606657377dc206baf3aafab9e764c9151b5e76c37537b42f5a4ffd932daf4133ad9b1e667310212e59fe5f3209a5d6f92c796c63088750a5bebd4cc01

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog
+
+## 0.1.0
+
+- First version, extracted from the onlylogs gem, where it keeps saved searches next to each log directory:
+  `include SqliteShards`, `database_migrations`, `with_database(path) { ... }`, `bound_to(path) { scope }` and
+  `clear_database_connections`.

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 Renuo AG
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,96 @@
+# SqliteShards
+
+A gem to connect your ActiveRecord models to a separate sqlite file.
+
+Some data does not belong into the app's main database. 
+For example, on [onlylogs](https://github.com/renuo/onlylogs) we persist saved queries into a small sqlite file, 
+without interfering with the main app database.
+
+## Usage
+
+Include `SqliteShards` in an abstract class and list its migrations:
+
+```ruby
+class NotesRecord < ActiveRecord::Base
+  self.abstract_class = true
+  include SqliteShards
+
+  database_migrations(
+    ->(connection) {
+      connection.create_table(:notes) do |t|
+        t.string :title, null: false
+        t.timestamps
+      end
+    },
+    ->(connection) { 
+      connection.add_column(:notes, :body, :text) 
+    }
+  )
+end
+
+class Note < NotesRecord
+end
+```
+
+Then pick a file:
+
+```ruby
+NotesRecord.with_database("storage/projects/acme/notes.db") do
+  Note.create!(title: "Hello")
+  Note.where(title: "Hello").count
+end
+```
+
+Inside the block every model under `NotesRecord` reads and writes that file. 
+Outside it, queries raise `ActiveRecord::ConnectionNotDefined`, so data can't end up in the wrong place by accident.
+
+Wrapping it in the object that owns the file reads well:
+
+```ruby
+class Project < ApplicationRecord
+  def with_database(create: true, &) = NotesRecord.with_database(root_path.join("notes.db"), create:, &)
+end
+
+project.with_database { Note.count }
+```
+
+### Without a block
+
+`bound_to` returns a relation you can keep, chain and write through outside a block. 
+Every call on it, and on the relations and records it returns, runs inside the database:
+
+```ruby
+class Project < ApplicationRecord
+  def notes = NotesRecord.bound_to(root_path.join("notes.db")) { Note.order(created_at: :desc) }
+end
+
+note = project.notes.create!(title: "Hello")
+note.update!(title: "Hi")
+project.notes.where(title: "Hi").pluck(:id)
+```
+
+The records it returns are wrappers that forward every call, so `note.class` is `SqliteShards::Bound`, not `Note`.
+
+### Options
+
+- `with_database(path, create: false)` yields nothing and returns `nil` when the file doesn't exist, instead of
+  creating it. Useful for reading without leaving empty databases behind.
+- `NotesRecord.transaction { ... }` inside `with_database` spans all models of that file: they share one connection.
+- `with_database` calls nest, for the same class or different ones.
+- `clear_database_connections` closes every connection the class opened.
+
+## Requirements
+
+Active Record 8.0 or newer and the `sqlite3` gem. 
+
+## Development
+
+```sh
+bin/setup
+bin/check      # tests
+bin/fastcheck  # standardrb
+```
+
+## License
+
+MIT, see [LICENSE.txt](LICENSE.txt).

data/lib/sqlite_shards/bound.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "delegate"
+
+module SqliteShards
+  # A relation or record whose every call runs inside its database; what those calls return is bound the same way.
+  class Bound < Delegator
+    def initialize(object, &connect)
+      super(object)
+      @connect = connect
+    end
+
+    def __getobj__ = @object
+
+    def __setobj__(object)
+      @object = object
+    end
+
+    def method_missing(name, *args, **kwargs, &block)
+      @connect.call { bind(@object.public_send(name, *args, **kwargs, &block)) }
+    end
+
+    def respond_to_missing?(name, include_private = false) = @object.respond_to?(name, include_private)
+
+    def ==(other)
+      @object == (other.is_a?(Bound) ? other.__getobj__ : other)
+    end
+
+    private
+
+    def bind(result)
+      case result
+      when ::ActiveRecord::Relation, ::ActiveRecord::Base then Bound.new(result, &@connect)
+      when ::Array then result.map { |element| bind(element) }
+      else result
+      end
+    end
+  end
+end

data/lib/sqlite_shards/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SqliteShards
+  VERSION = "0.1.0"
+end

data/lib/sqlite_shards.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "digest"
+require "fileutils"
+require_relative "sqlite_shards/version"
+require_relative "sqlite_shards/bound"
+
+# Keeps the models under an abstract class in SQLite files picked at runtime, a database per file:
+#
+#   class NotesRecord < ActiveRecord::Base
+#     self.abstract_class = true
+#     include SqliteShards
+#
+#     database_migrations ->(connection) { connection.create_table(:notes) { |t| t.string :title } }
+#   end
+#
+#   class Note < NotesRecord; end
+#
+#   NotesRecord.with_database("path/to/notes.db") { Note.all.to_a }
+#
+# Outside with_database the models have no connection, so Note.all raises. Their schema is known anyway, since every
+# file is migrated the same way: it is read from an in-memory database migrated like the files, so Note.new and a
+# controller's parameter wrapping work without a file.
+#
+# bound_to gives a relation that can be kept and used outside a block: every call on it, and on the relations and
+# records it returns, runs inside with_database.
+#
+#   notes = NotesRecord.bound_to("path/to/notes.db") { Note.all }
+#   note = notes.create!(title: "Hello")
+#   note.update!(title: "Hi")
+#
+# Each file is a shard of the class, connected on first use. A file deleted or replaced since (a purged or renamed
+# directory, say) has another inode and so a shard of its own: the old shard's idle connections are reaped, and a
+# connection in use, like a test transaction's, is never pulled away.
+#
+# Migrations are blocks run in order, tracked by SQLite's user_version: append new ones, never edit or reorder
+# shipped ones.
+module SqliteShards
+  extend ActiveSupport::Concern
+
+  class Error < StandardError; end
+
+  class UnavailableError < Error; end
+
+  CONNECTION_TIMEOUT = 5_000
+
+  included do
+    self.connection_class = true
+    self.connection_specification_name = name
+
+    @database_migrations = []
+    @database_shards = Set.new
+    @database_lock = Mutex.new
+  end
+
+  # Every file is migrated the same way, so the schema is known without one: see schema_cache.
+  SCHEMA_SHARD = :schema
+
+  class_methods do
+    def database_migrations(*migrations)
+      @database_migrations = migrations
+    end
+
+    # Without create, a file that does not exist yields nothing and returns nil.
+    def with_database(path, create: true, &)
+      path = ::File.expand_path(path.to_s)
+      if create
+        create_database_file(path)
+      elsif !::File.exist?(path)
+        return
+      end
+
+      connected_to(role: ActiveRecord.writing_role, shard: database_shard(path), &)
+    end
+
+    def bound_to(path, &scope)
+      connect = ->(&block) { with_database(path, &block) }
+      connect.call { Bound.new(scope.call, &connect) }
+    end
+
+    # Active Record reads a model's schema through these, all memoized on the class: the table and its primary key,
+    # the columns, and their types. Outside a file they read the in-memory schema database instead.
+    %i[schema_cache load_schema _default_attributes].each do |method|
+      define_method(method) do
+        return super() unless current_shard == default_shard
+
+        connection_class_for_self.send(:with_schema_database) { super() }
+      end
+    end
+
+    def clear_database_connections
+      @database_lock.synchronize do
+        @database_shards.each do |shard|
+          connection_handler.remove_connection_pool(connection_specification_name, role: ActiveRecord.writing_role, shard: shard)
+        end
+        @database_shards.clear
+      end
+    end
+
+    private
+
+    # An in-memory database is lost when the reaper closes its one connection, so it is migrated on every use;
+    # Active Record keeps the schema once read, so that is rare.
+    def with_schema_database(&)
+      @database_lock.synchronize do
+        unless @database_shards.include?(SCHEMA_SHARD)
+          connection_handler.establish_connection(
+            {adapter: "sqlite3", database: ":memory:", pool: 1},
+            owner_name: self, role: ActiveRecord.writing_role, shard: SCHEMA_SHARD
+          )
+          @database_shards << SCHEMA_SHARD
+        end
+      end
+
+      connected_to(role: ActiveRecord.writing_role, shard: SCHEMA_SHARD) do
+        connection_pool.with_connection { |connection| migrate_database(connection) }
+        yield
+      end
+    end
+
+    def create_database_file(path)
+      FileUtils.mkdir_p(::File.dirname(path))
+      FileUtils.touch(path) unless ::File.exist?(path)
+    rescue SystemCallError
+      raise UnavailableError, "#{::File.basename(path)} cannot be created"
+    end
+
+    def database_shard(path)
+      stat = ::File.stat(path)
+      # Digested, so the path never leaks into error messages.
+      shard = :"file_#{Digest::SHA256.hexdigest([path, stat.dev, stat.ino].join(":"))[0, 16]}"
+
+      @database_lock.synchronize do
+        unless @database_shards.include?(shard)
+          pool = connection_handler.establish_connection(
+            {adapter: "sqlite3", database: path, timeout: CONNECTION_TIMEOUT},
+            owner_name: self, role: ActiveRecord.writing_role, shard: shard
+          )
+          pool.with_connection { |connection| migrate_database(connection) }
+          @database_shards << shard
+        end
+      end
+
+      shard
+    end
+
+    # Transactions are IMMEDIATE, so the version is read under the write lock and
+    # concurrent workers cannot run the same migration twice.
+    def migrate_database(connection)
+      return if database_version(connection) >= @database_migrations.size
+
+      connection.transaction do
+        version = database_version(connection)
+        @database_migrations.drop(version).each { |migration| migration.call(connection) }
+        connection.execute("PRAGMA user_version = #{@database_migrations.size}")
+      end
+    end
+
+    def database_version(connection)
+      connection.select_value("PRAGMA user_version").to_i
+    end
+  end
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: sqlite_shards
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Alessandro Rodi
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '8.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '8.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: Include SqliteShards in an abstract Active Record class and its models
+  read and write the SQLite file you pass to with_database, each file a shard of its
+  own, migrated on first use.
+email:
+- alessandro.rodi@renuo.ch
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/sqlite_shards.rb
+- lib/sqlite_shards/bound.rb
+- lib/sqlite_shards/version.rb
+homepage: https://github.com/renuo/sqlite_shards
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/renuo/sqlite_shards
+  source_code_uri: https://github.com/renuo/sqlite_shards
+  changelog_uri: https://github.com/renuo/sqlite_shards/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.21
+specification_version: 4
+summary: Active Record models whose data lives in SQLite files picked at runtime,
+  one database per file.
+test_files: []