activerecord-virgodb-adapter 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2506b4fecd647493533cf94951776ebbc4acef508a1bbf1a846001228cfc4bb2
-  data.tar.gz: 546b407b74267d0fbb6792fcfa26041da4f078f4c241e96919a7b89c3a4525f2
+  metadata.gz: a674e66bbf9674502c2c5cb25ec4a1dfeca0f9201fa001a54c5a03f025b09718
+  data.tar.gz: 32560701fb69a497e6c539cd71e9de071423523f00d2baa1333b6328973b5da0
 SHA512:
-  metadata.gz: 9f8c46262072bd70640666b45f9254902100f6bbde82af984655f5720131227119c768eb1d24bfc8fc7b0435d067e6438ab4b89eb835d5bf7b6664978218bf50
-  data.tar.gz: 12a8d38269b32b6af45e9dd637a47e1b9c74daa9419dea208a92c9f089bfa95b80630039c670399a9d515d03ca2470d01e7ac07515c0e21950f269025114aac2
+  metadata.gz: e9b192ed7e8bc0ad7e3b18a6ecdda847c323a272f70c35b236685718b0aa0b27cca1b63195c4a6df2e403cb49e017ff455f29e8b0a1b932067fcc9cef64c63fc
+  data.tar.gz: caeefe677d520080a4db26d04852dbc7adfaa2667c710cc195bd4877307a7ccf452a86b7ff6700d618b67700164217e34676673763c3aad07d056f8d8a4e6693

data/lib/active_record/connection_adapters/virgodb_adapter.rb

@@ -0,0 +1,373 @@
+# frozen_string_literal: true
+
+require "active_record/connection_adapters/sqlite3_adapter"
+require "bigdecimal"
+require "json"
+
+module ActiveRecord
+  module ConnectionAdapters
+    # ActiveRecord adapter for virgodb migrations -- lets any Rails app
+    # define and evolve a virgodb table's schema the same way it already
+    # does for SQLite (bin/rails db:migrate): real ActiveRecord::Migration
+    # subclasses, raw type strings, picked up by `rails db:migrate`
+    # alongside every other database.
+    #
+    # virgodb's manifest is already a real SQLite file, so subclassing
+    # SQLite3Adapter (not building from AbstractAdapter) gets
+    # structure_dump/structure_load -- the :sql schema format -- for free,
+    # zero custom dumper code. Only create_table/add_column need
+    # overriding: call `super` for a real, if never-written-to, SQLite
+    # table (SQLite's type-name grammar accepts arbitrary bare-word type
+    # declarations -- verified against a real connection, see
+    # crates/compact/src/schema.rs's doc comment for the exact grammar and
+    # why it has to be `"AnyLast String updated_at"`, not
+    # `"AnyLast(String, updated_at)"`), and additionally record the column
+    # in a `schema_versions` table matching exactly what
+    # `compact::schema_from_columns` on the Rust side expects to parse.
+    #
+    # One manifest can back several virgodb tables sharing a single
+    # `database.yml` connection and migrations path -- the same way a host
+    # app's `clickhouse:` connection might already hold all of its ClickHouse
+    # tables under one connection. `schema_versions` (and the Rust-side
+    # manifest's `run_files`/`inline_batches`/`orphaned_files`/
+    # `maintenance_lease` catalog tables) are scoped by `table_name`
+    # specifically to make this safe: each logical table's schema history
+    # and physical run files stay independent even though they live in the
+    # same SQLite file.
+    #
+    # Additive-only, matching `Manifest::register_schema_version`'s own
+    # rule and virgodb's immutable-Parquet philosophy: `remove_column`,
+    # `rename_column`, and `change_column` are all disabled outright rather
+    # than silently letting the real SQLite table drift out of sync with
+    # what `schema_versions` describes.
+
+    # virgodb's raw "Timestamp" type (crates/compact/src/schema.rs) is plain
+    # UNIX seconds -- an Integer, not a real SQL DATETIME/TIMESTAMP string --
+    # but it still matches Rails' generic %r(time)i registration (see
+    # VirgodbAdapter.initialize_type_map below), landing on the stock
+    # ActiveRecord::Type::DateTime, which only knows how to parse a real
+    # date/time STRING (or an already-real Time/Date/Hash). Two confirmed,
+    # real failure modes from that mismatch, not just cosmetic ones:
+    #
+    # 1. `Type::DateTime.new.cast("0")` -- "0" being a perfectly valid
+    #    epoch-seconds default (e.g. a real host app declaring `DEFAULT 0`
+    #    as a sentinel on a nullable-in-spirit timestamp column)
+    #    -- returns nil. `Column#default` itself is unaffected (DateTime is
+    #    `mutable?`, so `Column#initialize` keeps the raw string rather than
+    #    deserializing it -- see ActiveModel::Type::DateTime#mutable?'s own
+    #    comment), which is why the schema_versions capture above was never
+    #    wrong. But `Model.column_defaults` (what a real host app's own
+    #    schema-annotation tooling reads) DOES deserialize through the cast
+    #    type, surfacing as a misleading `default(NULL)` in a real model's
+    #    annotation for a column that has a perfectly real default.
+    # 2. `Type::DateTime.new.cast(1753000000)` (a real Integer write-time
+    #    value) ALSO doesn't produce a real Time object -- DateTime#cast
+    #    only recognizes String/Time/Date/Hash input, so an Integer just
+    #    passes through unchanged. Harmless today (nothing in this
+    #    codebase reads a Timestamp column back through a real
+    #    ActiveRecord attribute -- see Virgodb::Registry.query, which
+    #    returns raw parsed JSON, bypassing AR type casting entirely), but
+    #    a real latent gap the moment anything ever does.
+    #
+    # Subclassing ActiveRecord::Type::DateTime (not building from scratch)
+    # keeps `.type == :datetime` -- the semantically correct label for a
+    # timestamp column, unlike the columns String fix above where Rails had
+    # no keyword at all -- and keeps every other DateTime behavior (real
+    # date strings, Time/Date input, timezone handling) working exactly as
+    # before; only the epoch-seconds-as-Integer-or-String case needed a
+    # fix. `deserialize` isn't overridden separately -- ActiveModel::Type
+    # ::Value#deserialize just calls `cast` by default (confirmed: neither
+    # ActiveModel::Type::DateTime nor ActiveRecord::Type::DateTime override
+    # it), so fixing `cast` alone is enough for both directions.
+    class VirgodbTimestamp < ActiveRecord::Type::DateTime
+      def cast(value)
+        return value if value.nil? || value.is_a?(::Time) || value.is_a?(::DateTime)
+
+        seconds = Float(value, exception: false)
+        seconds ? ::Time.at(seconds).utc : super
+      end
+
+      def serialize(value)
+        value.respond_to?(:to_time) ? value.to_time.to_i : value
+      end
+    end
+
+    class VirgodbAdapter < SQLite3Adapter
+      ADAPTER_NAME = "Virgodb"
+
+      # Real bug, found (not assumed) by testing: setting `auto_vacuum =
+      # INCREMENTAL` from `create_table` -- even from create_table's very
+      # FIRST call on a brand-new manifest, before any table exists -- was
+      # already too late. SQLite3Adapter's own `configure_connection` sets
+      # `journal_mode = wal` via DEFAULT_PRAGMAS (`super`, below) as part of
+      # establishing the connection itself, which happens before this
+      # adapter's `create_table` ever runs -- and switching journal_mode to
+      # WAL requires SQLite to actually write to the file (creating the -wal
+      # file, an implicit checkpoint), which "poisons" the database from
+      # auto_vacuum's perspective the same way any other content would.
+      # Confirmed directly: a manual `PRAGMA auto_vacuum = INCREMENTAL`
+      # issued immediately after `establish_connection`, before ANY table or
+      # migration ran, still silently failed. The pragma has to be set
+      # before `super` runs here, not merely before the first CREATE TABLE.
+      #
+      # Second real bug, found the same way: `configure_connection` runs on
+      # EVERY new raw connection this adapter's pool ever opens, not just
+      # the manifest's first one -- so an unconditional `auto_vacuum =`
+      # unconditionally re-issued `PRAGMA auto_vacuum='2'` on every single
+      # one of them, forever. Once a manifest is genuinely in INCREMENTAL
+      # mode (the common case for any real, already-migrated manifest),
+      # re-setting it to that SAME value is not the free no-op re-setting an
+      # already-NONE manifest is: confirmed directly with a raw SQLite3
+      # connection, re-issuing the SET pragma while a second connection
+      # holds an open `BEGIN IMMEDIATE` write transaction (exactly what
+      # virgodb's own maintenance thread does while compacting, see
+      # crates/compact/src/maintenance.rs) blocks for the full busy_timeout
+      # and then raises SQLite3::BusyException -- surfacing in a host app as
+      # an ActiveRecord::StatementTimeout on whatever request happened to
+      # need a brand-new pooled connection at that moment. A bare read
+      # (`PRAGMA auto_vacuum` with no argument) does NOT block the same way,
+      # confirmed the same way -- so skip the write once it's already
+      # reached the target value instead of paying that contention risk on
+      # every connection after the first.
+      def configure_connection
+        if @raw_connection.respond_to?(:auto_vacuum=) && @raw_connection.auto_vacuum != 2
+          @raw_connection.auto_vacuum = "incremental"
+        end
+        super
+      end
+
+      class << self
+        private
+
+        # Real bug, found (not assumed) by a host app trying to build real
+        # ActiveRecord models on top of virgodb tables: `Column#type` (what
+        # every host app actually reads -- annotation generators, `cast`,
+        # attribute serialization) comes from `TYPE_MAP.lookup(sql_type)`,
+        # and Rails' own %r(char)i / %r(int)i / %r(date)i / etc. regexes
+        # (registered by AbstractAdapter#initialize_type_map, SQLite3Adapter
+        # only adds `int`) exist to recognize real SQL DDL keywords like
+        # "varchar" or "datetime" -- never anything from virgodb's own raw
+        # type vocabulary (crates/compact/src/schema.rs), since `t.column
+        # name, "raw string"` bypasses Rails' normal :string/:datetime/etc
+        # -> DDL-keyword translation entirely. Two confirmed failure modes:
+        #
+        # 1. A bare "String" or "LowCardinality String" column matches NONE
+        #    of Rails' base regexes (there's no keyword for it -- real
+        #    SQLite migrations only ever produce "varchar"), so `.type`
+        #    falls through to `Type.default_value`, an untyped
+        #    `ActiveModel::Type::Value` whose `.type` is nil.
+        # 2. An `AnyLast <type> <order_by_column>` column (the two-scalar-
+        #    column tuple-workaround pairs a real host app might use to
+        #    emulate ClickHouse-style last-touch columns) can get a WRONG
+        #    match instead of no match: Rails'
+        #    %r(date)i is an unanchored substring regex, and
+        #    "AnyLast String updated_at" contains the substring "date"
+        #    (from "upDATEd_at") purely by accident, mis-casting a String
+        #    column as :date. Confirmed directly: TYPE_MAP.lookup("AnyLast
+        #    String updated_at").class == ActiveRecord::Type::Date.
+        #
+        # The fix strips virgodb's own wrapper syntax down to the bare base
+        # type and re-enters `m.lookup` for it, rather than trying to write
+        # an ever-more-specific whole-string regex per aggregate function x
+        # base type combination -- that would still be one accidental
+        # trailing-column-name substring match away from the same bug the
+        # moment a future order-by column happens to contain "int"/"char"/
+        # "time"/etc. `AnyLast` is the only aggregate function that ever
+        # carries a trailing order-by column (it's the one that needs to
+        # know which row is "last"; Any/Sum/Max/Min/UniqueArrayUnion are
+        # simple scalar aggregates with no ordering to track), so it's the
+        # only one that needs the extra trailing-token strip.
+        def initialize_type_map(m)
+          super
+
+          register_class_with_limit m, /\A(?:LowCardinality\s+)?String\z/i, Type::String
+          m.register_type(/\ATimestamp\z/i, VirgodbTimestamp.new)
+
+          m.register_type(/\AAnyLast\s+/i) do |sql_type|
+            base = sql_type.sub(/\AAnyLast\s+/i, "").sub(/\s+\S+\z/, "")
+            m.lookup(base)
+          end
+
+          m.register_type(/\A(?:Any|Sum|Max|Min|UniqueArrayUnion)\s+/i) do |sql_type|
+            base = sql_type.sub(/\A(?:Any|Sum|Max|Min|UniqueArrayUnion)\s+/i, "")
+            m.lookup(base)
+          end
+        end
+      end
+
+      # `type_map` (AbstractAdapter) reads `self.class::TYPE_MAP` -- a plain
+      # constant lookup, not a method call -- so merely overriding
+      # `initialize_type_map` above does nothing on its own: without a
+      # `TYPE_MAP` constant of ITS OWN, `VirgodbAdapter::TYPE_MAP` constant
+      # lookup just walks up to the already-frozen `SQLite3Adapter::TYPE_MAP`
+      # (built at THAT class's load time, from the un-overridden method).
+      # Confirmed directly: the override above alone left every virgodb
+      # column's `.type` unchanged. SQLite3Adapter builds its own `TYPE_MAP`
+      # constant the exact same way (see its `TYPE_MAP = Type::TypeMap.new
+      # .tap { |m| initialize_type_map(m) }` line) specifically so subclasses
+      # can shadow it like this.
+      TYPE_MAP = Type::TypeMap.new.tap { |m| initialize_type_map(m) }
+
+      def create_table(table_name, **options, &block)
+        result = super
+        record_schema_version!(table_name)
+        result
+      end
+
+      def add_column(table_name, column_name, type, **options)
+        result = super
+        record_added_column!(table_name, column_name, type, options)
+        result
+      end
+
+      def remove_column(table_name, column_name, type = nil, **options)
+        raise NotImplementedError, "virgodb schemas are additive-only -- columns can be added, never removed (see schema_versions)"
+      end
+
+      def rename_column(table_name, column_name, new_column_name)
+        raise NotImplementedError, "virgodb schemas are additive-only -- columns can't be renamed once created"
+      end
+
+      def change_column(table_name, column_name, type, **options)
+        raise NotImplementedError, "virgodb schemas are additive-only -- an existing column's type can't change"
+      end
+
+      private
+
+      # Reads the table back via ActiveRecord's own column introspection --
+      # `sql_type` round-trips the raw virgodb type string verbatim, proven
+      # against a real SQLite connection immediately after a fresh CREATE
+      # TABLE -- and writes a new `schema_versions` row: the full current
+      # column list, versioned one higher than whatever was there before FOR
+      # THIS table_name. Mirrors Manifest::register_schema_version's shape
+      # exactly (same table_name scoping, same column names, same JSON shape
+      # for `columns`) so the Rust side can read whatever Ruby wrote with
+      # zero translation. Scoping `MAX(version)` by `table_name` is what makes
+      # it safe for several logical tables to share one manifest/connection:
+      # without it, the second table's first migration would start at
+      # version 2 (or worse, silently share version numbers with an
+      # unrelated table). Rails' OWN bookkeeping tables (`schema_migrations`,
+      # `ar_internal_metadata`) go through this SAME overridden `create_table`
+      # the first time either is lazily created -- there's nothing
+      # virgodb-specific about them, and recording a schema_versions row for
+      # them is pure noise (found while dumping a real structure.sql and
+      # seeing them show up as rows next to the real logical tables). Guarded
+      # here, not by checking `table_name` against `TABLES` in some caller --
+      # this adapter has no such registry, `create_table`'s own argument is
+      # the one place that reliably sees every table, virgodb or not.
+      #
+      # ONLY safe for `create_table` (a brand-new table with no prior
+      # history). NOT reused by `add_column` -- see `record_added_column!`'s
+      # own doc comment for the real bug this avoids: `columns(table_name)`
+      # stops reliably returning the ORIGINAL declared type text for
+      # pre-existing columns once the table has been through certain ALTER
+      # operations.
+      RAILS_INTERNAL_TABLES = %w[schema_migrations ar_internal_metadata].freeze
+
+      def record_schema_version!(table_name)
+        return if RAILS_INTERNAL_TABLES.include?(table_name.to_s)
+
+        ensure_schema_versions_table!
+
+        cols = columns(table_name).map do |c|
+          { "name" => c.name, "virgodb_type" => c.sql_type, "nullable" => c.null, "default" => stringify_default(c.default) }
+        end
+
+        insert_schema_version!(table_name, cols)
+      end
+
+      # A migration's `default:` (e.g. `t.column :clid, "String", null:
+      # false, default: ""`) round-trips through `columns(table_name).first
+      # .default` as a real, correctly-typed Ruby value now (String/Integer/
+      # BigDecimal) -- only possible since the TYPE_MAP fix above; before it,
+      # every virgodb String column's `.type` was nil/wrong and `.default`
+      # came back nil regardless of what the migration declared. schema_versions
+      # itself only stores plain JSON strings (matching `virgodb_type`'s own
+      # convention of storing raw text the Rust side parses, not a typed
+      # value) -- the Rust write path (crates/virgodb_ruby's row_convert.rs)
+      # re-parses this string against the column's real Arrow type at write
+      # time. `BigDecimal#to_s` explicitly forced to fixed-point ("F") rather
+      # than its default format, which switches to scientific notation for
+      # some magnitudes -- avoids writing e.g. "0.12345e3" into
+      # schema_versions where a human reading a real dump would expect
+      # "123.45" (Rust's f64 parser would actually accept either form, this
+      # is purely for a human reading a real structure.sql/schema_versions
+      # dump, not a correctness requirement).
+      def stringify_default(value)
+        return nil if value.nil?
+        value.is_a?(BigDecimal) ? value.to_s("F") : value.to_s
+      end
+
+      # `add_column`'s schema_versions update -- deliberately does NOT
+      # re-introspect the whole table via `columns(table_name)` the way
+      # `record_schema_version!` does. Real bug, found (not assumed) by
+      # testing: `super`'s real `ALTER TABLE ADD COLUMN` -- specifically for
+      # a `null: false` column, which SQLite's native ALTER can't add without
+      # a default -- makes Rails' SQLite3Adapter fall back to its "rebuild
+      # the table" strategy (copy to a temp table, drop, rename). Confirmed
+      # directly: introspecting the table immediately afterward, even via a
+      # completely fresh connection/process (so not a Rails-process-level
+      # schema cache artifact), returns Rails' own GENERIC type names for
+      # every PRE-EXISTING column ("INTEGER"/"datetime"/"" instead of
+      # "Int64"/"Timestamp"/"LowCardinality String") -- the rebuild
+      # reconstructs the physical table's DDL from Rails' normalized column
+      # metadata, silently discarding the original literal type text for
+      # every column it didn't just add. Re-deriving the full column list
+      # via introspection after ANY add_column would therefore corrupt every
+      # already-correct entry in `schema_versions`.
+      #
+      # The fix: never re-derive what's already known to be correct. Read
+      # the CURRENT (highest-version) column list straight out of
+      # `schema_versions` itself (the one thing guaranteed not to have been
+      # touched by the ALTER), and append only the ONE new column -- whose
+      # name/type/nullability are already fully known from `add_column`'s own
+      # arguments, no introspection needed for it either.
+      def record_added_column!(table_name, column_name, type, options)
+        return if RAILS_INTERNAL_TABLES.include?(table_name.to_s)
+
+        ensure_schema_versions_table!
+
+        current_json = select_value("SELECT columns FROM schema_versions WHERE table_name = #{quote(table_name)} ORDER BY version DESC LIMIT 1")
+        current_cols = current_json ? JSON.parse(current_json) : []
+
+        new_col = { "name" => column_name.to_s, "virgodb_type" => type.to_s, "nullable" => options[:null] != false, "default" => stringify_default(options[:default]) }
+        insert_schema_version!(table_name, current_cols + [ new_col ])
+      end
+
+      def insert_schema_version!(table_name, cols)
+        next_version = (select_value("SELECT MAX(version) FROM schema_versions WHERE table_name = #{quote(table_name)}") || 0).to_i + 1
+
+        execute(<<~SQL)
+          INSERT INTO schema_versions (table_name, version, columns, applied_at)
+          VALUES (#{quote(table_name)}, #{next_version}, #{quote(cols.to_json)}, #{Time.now.to_i})
+        SQL
+      end
+
+      # Same DDL as Manifest::open()'s bootstrap on the Rust side (same
+      # table/column names, including the `table_name` scoping column) --
+      # IF NOT EXISTS makes this safe to call whether virgodb's own Rust
+      # code or this Ruby adapter created the manifest file first. Plain
+      # INTEGER PRIMARY KEY, not AUTOINCREMENT -- see the matching comment
+      # in manifest/src/lib.rs for why (breaks a real structure_dump/
+      # structure_load round trip the moment any host app's
+      # ActiveRecord::SchemaDumper.ignore_tables is non-empty).
+      def ensure_schema_versions_table!
+        execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS schema_versions (
+            id         INTEGER PRIMARY KEY,
+            table_name TEXT NOT NULL,
+            version    INTEGER NOT NULL,
+            columns    TEXT NOT NULL,
+            applied_at INTEGER NOT NULL
+          )
+        SQL
+      end
+    end
+  end
+end
+
+ActiveRecord::ConnectionAdapters.register(
+  "virgodb",
+  "ActiveRecord::ConnectionAdapters::VirgodbAdapter",
+  "activerecord-virgodb-adapter"
+)

data/lib/active_record/tasks/virgodb_database_tasks.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+require "active_record/tasks/sqlite_database_tasks"
+require "json"
+require "sqlite3"
+
+module ActiveRecord
+  module Tasks
+    # `ActiveRecord::Tasks::SQLiteDatabaseTasks#structure_dump` just shells
+    # out to the real `sqlite3` CLI (`.schema --nosys`, or a raw `SELECT sql
+    # FROM sqlite_master` when a host app has any
+    # `ActiveRecord::SchemaDumper.ignore_tables` configured -- e.g. excluding
+    # tables owned by a separate replication/backup mechanism) against the
+    # configured database file --
+    # DDL only. `ActiveRecord::Tasks::DatabaseTasks.dump_schema` then
+    # special-cases exactly one table on top of that: if `schema_migrations`
+    # exists, it appends that table's real row data (`connection
+    # .dump_schema_versions`) to the same dump file, because Rails' own
+    # migration bookkeeping needs those rows to survive a `db:schema:load`/
+    # `structure_load`, not just the table shape.
+    #
+    # `schema_versions` (this adapter's own bookkeeping table, plural,
+    # distinct from Rails' singular `schema_migrations`) needs exactly the
+    # same treatment and doesn't get it for free: it holds real data (the
+    # versioned column list `compact::schema_from_columns` on the Rust side
+    # depends on), not just structure, and Rails has no way to know that a
+    # table it didn't create needs the same row-preserving special case.
+    # Confirmed empirically: without this, a fresh database prepared via
+    # `structure_load` (which Rails does automatically whenever a `db:migrate`
+    # invocation for a later environment/connection finds an existing,
+    # version-matching dump already on disk from an earlier one -- not
+    # something this adapter can opt out of) ends up with the real
+    # host-app table but an EMPTY `schema_versions`, so
+    # `Manifest::current_schema_version()` on the Rust side returns `None`
+    # even though the SQLite table itself is correct.
+    class VirgodbDatabaseTasks < SQLiteDatabaseTasks
+      class << self
+        # Optional host-app hook: `->(sql_table_name) { {sort_by:, key_column:,
+        # partition_column:} or nil }`. Purely a documentation comment written
+        # above CREATE TABLE at dump time -- nothing on the Rust or Ruby side
+        # ever reads it back. This adapter gem stays host-app-agnostic (see
+        # the class doc comment above: "lets any Rails app..."), so it can't
+        # hardcode any one host app's sort_by/key_column config (that's
+        # app/services/virgodb/registry.rb's job, which already has this
+        # exact data for real -- passed to Virgodb.start_maintenance on every
+        # boot). A host app that wants it surfaced sets this once, e.g. in an
+        # initializer; left nil, structure_dump behaves exactly as before.
+        attr_accessor :table_layout_provider
+
+        # Optional host-app hook: `->(sql_table_name) { path_or_nil }`.
+        # Convention over configuration, not opt-in like table_layout_provider
+        # above: this gem is Rails-specific (see the class doc comment
+        # above -- "Lets any Rails app..."), so it bakes in the same
+        # convention Rails' own generators use for SQLite's storage location
+        # (config/database.yml's generated `storage/#{Rails.env}.sqlite3`)
+        # -- `storage/#{Rails.env}_virgodb_#{sql_table_name}_data`, right
+        # next to the manifest. A host app gets working data-dir cleanup
+        # with zero setup. One with its own directory convention calls
+        # `data_dir_provider =` to override; `= nil` opts every table out
+        # entirely, same as plain SQLiteDatabaseTasks -- distinguished from
+        # "never touched this accessor" via `defined?` below so an explicit
+        # opt-out can't be silently overridden by the default.
+        def data_dir_provider
+          defined?(@data_dir_provider) ? @data_dir_provider : DEFAULT_DATA_DIR_PROVIDER
+        end
+        attr_writer :data_dir_provider
+      end
+
+      DEFAULT_DATA_DIR_PROVIDER = lambda do |sql_table_name|
+        Rails.root.join("storage", "#{Rails.env}_virgodb_#{sql_table_name}_data").to_s
+      end
+
+      def structure_dump(filename, extra_flags)
+        super
+        reformat_create_table_statements!(filename)
+        annotate_physical_layout!(filename)
+        dump_schema_versions!(filename)
+      end
+
+      # Overridden (not just inherited from SQLiteDatabaseTasks) so dropping
+      # the manifest also cleans up whatever data_dir_provider says each of
+      # its tables' physical files live under. db:drop only ever knew about
+      # the ONE file database.yml declares -- left alone, it silently
+      # orphans every Parquet run file and Tier 1 inline-batch pointer file
+      # forever, since there's no catalog left afterward to even reference
+      # them (compact::vacuum, which cleans those up via the manifest's own
+      # orphaned_files table, can't reach them either once the manifest's
+      # gone).
+      #
+      # Reads which tables exist via a fresh, read-only, raw SQLite3
+      # connection (not `connection`/`ActiveRecord::Base.lease_connection`
+      # -- unlike structure_dump, drop doesn't establish one for this
+      # specific db_config, and this shouldn't depend on whatever happens
+      # to be currently leased) BEFORE calling super, since super deletes
+      # the manifest file -- schema_versions goes with it.
+      def drop
+        provider = self.class.data_dir_provider
+        table_names = provider ? table_names_for_data_dir_cleanup : []
+        super
+        table_names.each do |table_name|
+          dir = provider.call(table_name)
+          FileUtils.rm_rf(dir) if dir
+        end
+      end
+
+      private
+
+      def table_names_for_data_dir_cleanup
+        db_path = db_config.database
+        path = File.absolute_path?(db_path) ? db_path : File.join(root, db_path)
+        return [] unless File.exist?(path)
+
+        raw = SQLite3::Database.new(path, readonly: true)
+        raw.busy_timeout = 5_000
+        raw.execute("SELECT DISTINCT table_name FROM schema_versions").flatten
+      rescue SQLite3::SQLException
+        # File.exist? isn't proof schema_versions exists -- SQLite
+        # auto-creates an empty file the instant something connects to a
+        # path that doesn't exist yet, and Rails' parallel-test setup
+        # (ActiveRecord::TestDatabases.create_and_load_schema, one file per
+        # worker) does exactly that via schema_up_to_date? moments before
+        # calling drop. Nothing to clean up in that case.
+        []
+      ensure
+        raw&.close
+      end
+
+      # Only the LATEST schema_versions row per table_name is dumped.
+      # `Manifest::current_schema_version` (manifest/src/lib.rs) only ever
+      # reads `ORDER BY version DESC LIMIT 1` -- confirmed directly, no Rust
+      # or Ruby code anywhere reads an older version row, so keeping full
+      # history here isn't a correctness requirement. It also isn't a
+      # readability win: each version's `columns` re-lists every column from
+      # scratch, so N migrations means N near-duplicate JSON blobs, longer
+      # every time. The version count/dates are still worth one line for
+      # orientation -- the full column list per version isn't; the
+      # migration files themselves (db/migrate_virgodb_*/*.rb, timestamped,
+      # under git blame) are already the authoritative "what changed when"
+      # record. `register_schema_version`'s additive-only check is
+      # unaffected: it only ever compares a new migration's proposed columns
+      # against the CURRENT version, which is exactly what's still dumped.
+      def dump_schema_versions!(filename)
+        return unless connection.data_source_exists?("schema_versions")
+
+        rows = connection.select_all("SELECT id, table_name, version, columns, applied_at FROM schema_versions ORDER BY table_name, version")
+        return if rows.to_a.empty?
+
+        File.open(filename, "a") do |f|
+          f.puts
+          rows.to_a.group_by { |r| r["table_name"] }.each do |table_name, versions|
+            latest = versions.max_by { |r| r["version"] }
+            history = versions.map { |r| "v#{r['version']} (#{Time.at(r['applied_at'].to_i).utc.strftime('%Y-%m-%d')})" }.join(", ")
+
+            f.puts "-- #{table_name} schema history: #{history} -- only the current version is dumped below; see db/migrate_virgodb_*/ for the rest"
+            pretty_columns = JSON.pretty_generate(JSON.parse(latest["columns"]))
+            f.puts(
+              "INSERT INTO \"schema_versions\" (id, table_name, version, columns, applied_at) VALUES (\n" \
+              "  #{latest['id']}, #{connection.quote(table_name)}, #{latest['version']},\n" \
+              "  #{connection.quote(pretty_columns)},\n" \
+              "  #{latest['applied_at']}\n" \
+              ");"
+            )
+            f.puts
+          end
+        end
+      end
+
+      # Writes `table_layout_provider`'s sort_by/key_column/partition_column
+      # as a comment directly above each table's (already reformatted,
+      # one-column-per-line) CREATE TABLE -- the closest a SQLite-flavored
+      # dump can get to ClickHouse's own `ORDER BY`/`PARTITION BY` clauses
+      # being right there in the DDL. No-op (and no dump format change at
+      # all) when no provider is registered.
+      def annotate_physical_layout!(filename)
+        provider = self.class.table_layout_provider
+        return unless provider
+
+        content = File.read(filename)
+        content.gsub!(/^CREATE TABLE (IF NOT EXISTS )?"?(\w+)"? \(/) do
+          statement_start = Regexp.last_match(0)
+          table_name = Regexp.last_match(2)
+          layout = provider.call(table_name)
+          next statement_start unless layout
+
+          comment = [
+            "-- Physical layout (app/services/virgodb/registry.rb):",
+            "--   sort_by:          #{layout[:sort_by].join(', ')}",
+            "--   key_column:       #{layout[:key_column]} (run file min/max pruning bounds)",
+            "--   partition_column: #{layout[:partition_column]}"
+          ].join("\n")
+
+          "#{comment}\n#{statement_start}"
+        end
+        File.write(filename, content)
+      end
+
+      # `SQLite3Adapter#structure_dump`'s raw `SELECT sql FROM sqlite_master`
+      # path (used whenever `ActiveRecord::SchemaDumper.ignore_tables` is
+      # non-empty, e.g. excluding tables owned by a separate replication/
+      # backup mechanism) reproduces each
+      # CREATE TABLE exactly as ActiveRecord originally generated it --
+      # one long line, every column crammed together. Fine for SQLite to
+      # load back (structure_load doesn't care about whitespace), useless to
+      # a human reading the dump next to `db/clickhouse_structure.sql`
+      # (ClickHouse's own `SHOW CREATE TABLE` is naturally one-column-per-line,
+      # for comparison). Purely cosmetic, safe to do unconditionally: only
+      # rewrites the exact same statement with different whitespace, no DDL
+      # semantics change. Skips statements that already span multiple lines
+      # (e.g. this adapter's own hand-written `schema_versions` DDL, created
+      # via a real newline-containing heredoc) -- nothing to reformat there.
+      def reformat_create_table_statements!(filename)
+        content = File.read(filename)
+        content.gsub!(/^CREATE TABLE (IF NOT EXISTS )?("[\w]+"|\w+) \((.*)\);$/) do
+          if_not_exists = Regexp.last_match(1)
+          table_name = Regexp.last_match(2)
+          columns = split_top_level_commas(Regexp.last_match(3))
+          next Regexp.last_match(0) if columns.size <= 1
+
+          "CREATE TABLE #{if_not_exists}#{table_name} (\n" + columns.map { |c| "  #{c.strip}" }.join(",\n") + "\n);"
+        end
+        File.write(filename, content)
+      end
+
+      # Splits a CREATE TABLE column list on commas that are NOT nested
+      # inside a type's own parens (e.g. `Decimal(18,2)`) -- a plain
+      # `String#split(",")` would wrongly cut `Decimal(18` and `2)` apart.
+      def split_top_level_commas(str)
+        parts = []
+        depth = 0
+        current = +""
+        str.each_char do |ch|
+          case ch
+          when "(" then depth += 1; current << ch
+          when ")" then depth -= 1; current << ch
+          when ","
+            depth.zero? ? (parts << current; current = +"") : current << ch
+          else
+            current << ch
+          end
+        end
+        parts << current unless current.strip.empty?
+        parts
+      end
+    end
+  end
+end
+
+# Registering our own subclass (not plain SQLiteDatabaseTasks) so
+# schema_versions' row data survives the dump/load round trip -- see the
+# class doc comment above for why a real Rails app can't avoid hitting this.
+ActiveRecord::Tasks::DatabaseTasks.register_task(/virgodb/, "ActiveRecord::Tasks::VirgodbDatabaseTasks")

data/lib/activerecord-virgodb-adapter.rb

@@ -1,392 +1,15 @@
 # frozen_string_literal: true
 
 # Lives at lib/activerecord-virgodb-adapter.rb (matching the gem name
-# exactly), not the more conventional-looking
-# lib/active_record/connection_adapters/virgodb_adapter.rb, so Bundler's
-# default require (`require gem_name`, no `require:` option needed in the
-# Gemfile) actually finds it. Rails' own ActiveRecord::ConnectionAdapters
-# .resolve treats the path passed to .register below as an opaque string,
-# not a real convention -- it's only ever exercised as a fallback if this
-# file somehow loads without Bundler's default require path having run
-# first, so it just points at wherever this file actually lives.
-require "active_record/connection_adapters/sqlite3_adapter"
-require "json"
-
-module ActiveRecord
-  module ConnectionAdapters
-    # ActiveRecord adapter for virgodb migrations -- lets any Rails app
-    # define and evolve a virgodb table's schema the same way it already
-    # does for SQLite (bin/rails db:migrate) and ClickHouse (the
-    # clickhouse-activerecord gem): real ActiveRecord::Migration subclasses,
-    # raw type strings, picked up by `rails db:migrate` alongside every
-    # other database.
-    #
-    # virgodb's manifest is already a real SQLite file, so subclassing
-    # SQLite3Adapter (not building from AbstractAdapter) gets
-    # structure_dump/structure_load -- the :sql schema format -- for free,
-    # zero custom dumper code. Only create_table/add_column need
-    # overriding: call `super` for a real, if never-written-to, SQLite
-    # table (SQLite's type-name grammar accepts arbitrary bare-word type
-    # declarations -- verified against a real connection, see
-    # crates/compact/src/schema.rs's doc comment for the exact grammar and
-    # why it has to be `"AnyLast String updated_at"`, not
-    # `"AnyLast(String, updated_at)"`), and additionally record the column
-    # in a `schema_versions` table matching exactly what
-    # `compact::schema_from_columns` on the Rust side expects to parse.
-    #
-    # One manifest can back several virgodb tables sharing a single
-    # `database.yml` connection and migrations path -- the same way a host
-    # app's `clickhouse:` connection might already hold all of its ClickHouse
-    # tables under one connection. `schema_versions` (and the Rust-side
-    # manifest's `run_files`/`inline_batches`/`orphaned_files`/
-    # `maintenance_lease` catalog tables) are scoped by `table_name`
-    # specifically to make this safe: each logical table's schema history
-    # and physical run files stay independent even though they live in the
-    # same SQLite file.
-    #
-    # Additive-only, matching `Manifest::register_schema_version`'s own
-    # rule and virgodb's immutable-Parquet philosophy: `remove_column`,
-    # `rename_column`, and `change_column` are all disabled outright rather
-    # than silently letting the real SQLite table drift out of sync with
-    # what `schema_versions` describes.
-    class VirgodbAdapter < SQLite3Adapter
-      ADAPTER_NAME = "Virgodb"
-
-      # Real bug, found (not assumed) by testing: setting `auto_vacuum =
-      # INCREMENTAL` from `create_table` -- even from create_table's very
-      # FIRST call on a brand-new manifest, before any table exists -- was
-      # already too late. SQLite3Adapter's own `configure_connection` sets
-      # `journal_mode = wal` via DEFAULT_PRAGMAS (`super`, below) as part of
-      # establishing the connection itself, which happens before this
-      # adapter's `create_table` ever runs -- and switching journal_mode to
-      # WAL requires SQLite to actually write to the file (creating the -wal
-      # file, an implicit checkpoint), which "poisons" the database from
-      # auto_vacuum's perspective the same way any other content would.
-      # Confirmed directly: a manual `PRAGMA auto_vacuum = INCREMENTAL`
-      # issued immediately after `establish_connection`, before ANY table or
-      # migration ran, still silently failed. The pragma has to be set
-      # before `super` runs here, not merely before the first CREATE TABLE.
-      def configure_connection
-        @raw_connection.auto_vacuum = "incremental" if @raw_connection.respond_to?(:auto_vacuum=)
-        super
-      end
-
-      def create_table(table_name, **options, &block)
-        result = super
-        record_schema_version!(table_name)
-        result
-      end
-
-      def add_column(table_name, column_name, type, **options)
-        result = super
-        record_added_column!(table_name, column_name, type, options)
-        result
-      end
-
-      def remove_column(table_name, column_name, type = nil, **options)
-        raise NotImplementedError, "virgodb schemas are additive-only -- columns can be added, never removed (see schema_versions)"
-      end
-
-      def rename_column(table_name, column_name, new_column_name)
-        raise NotImplementedError, "virgodb schemas are additive-only -- columns can't be renamed once created"
-      end
-
-      def change_column(table_name, column_name, type, **options)
-        raise NotImplementedError, "virgodb schemas are additive-only -- an existing column's type can't change"
-      end
-
-      private
-
-      # Reads the table back via ActiveRecord's own column introspection --
-      # `sql_type` round-trips the raw virgodb type string verbatim, proven
-      # against a real SQLite connection immediately after a fresh CREATE
-      # TABLE -- and writes a new `schema_versions` row: the full current
-      # column list, versioned one higher than whatever was there before FOR
-      # THIS table_name. Mirrors Manifest::register_schema_version's shape
-      # exactly (same table_name scoping, same column names, same JSON shape
-      # for `columns`) so the Rust side can read whatever Ruby wrote with
-      # zero translation. Scoping `MAX(version)` by `table_name` is what makes
-      # it safe for several logical tables to share one manifest/connection:
-      # without it, the second table's first migration would start at
-      # version 2 (or worse, silently share version numbers with an
-      # unrelated table). Rails' OWN bookkeeping tables (`schema_migrations`,
-      # `ar_internal_metadata`) go through this SAME overridden `create_table`
-      # the first time either is lazily created -- there's nothing
-      # virgodb-specific about them, and recording a schema_versions row for
-      # them is pure noise (found while dumping a real structure.sql and
-      # seeing them show up as rows next to the real logical tables). Guarded
-      # here, not by checking `table_name` against `TABLES` in some caller --
-      # this adapter has no such registry, `create_table`'s own argument is
-      # the one place that reliably sees every table, virgodb or not.
-      #
-      # ONLY safe for `create_table` (a brand-new table with no prior
-      # history). NOT reused by `add_column` -- see `record_added_column!`'s
-      # own doc comment for the real bug this avoids: `columns(table_name)`
-      # stops reliably returning the ORIGINAL declared type text for
-      # pre-existing columns once the table has been through certain ALTER
-      # operations.
-      RAILS_INTERNAL_TABLES = %w[schema_migrations ar_internal_metadata].freeze
-
-      def record_schema_version!(table_name)
-        return if RAILS_INTERNAL_TABLES.include?(table_name.to_s)
-
-        ensure_schema_versions_table!
-
-        cols = columns(table_name).map do |c|
-          { "name" => c.name, "virgodb_type" => c.sql_type, "nullable" => c.null }
-        end
-
-        insert_schema_version!(table_name, cols)
-      end
-
-      # `add_column`'s schema_versions update -- deliberately does NOT
-      # re-introspect the whole table via `columns(table_name)` the way
-      # `record_schema_version!` does. Real bug, found (not assumed) by
-      # testing: `super`'s real `ALTER TABLE ADD COLUMN` -- specifically for
-      # a `null: false` column, which SQLite's native ALTER can't add without
-      # a default -- makes Rails' SQLite3Adapter fall back to its "rebuild
-      # the table" strategy (copy to a temp table, drop, rename). Confirmed
-      # directly: introspecting the table immediately afterward, even via a
-      # completely fresh connection/process (so not a Rails-process-level
-      # schema cache artifact), returns Rails' own GENERIC type names for
-      # every PRE-EXISTING column ("INTEGER"/"datetime"/"" instead of
-      # "Int64"/"Timestamp"/"LowCardinality String") -- the rebuild
-      # reconstructs the physical table's DDL from Rails' normalized column
-      # metadata, silently discarding the original literal type text for
-      # every column it didn't just add. Re-deriving the full column list
-      # via introspection after ANY add_column would therefore corrupt every
-      # already-correct entry in `schema_versions`.
-      #
-      # The fix: never re-derive what's already known to be correct. Read
-      # the CURRENT (highest-version) column list straight out of
-      # `schema_versions` itself (the one thing guaranteed not to have been
-      # touched by the ALTER), and append only the ONE new column -- whose
-      # name/type/nullability are already fully known from `add_column`'s own
-      # arguments, no introspection needed for it either.
-      def record_added_column!(table_name, column_name, type, options)
-        return if RAILS_INTERNAL_TABLES.include?(table_name.to_s)
-
-        ensure_schema_versions_table!
-
-        current_json = select_value("SELECT columns FROM schema_versions WHERE table_name = #{quote(table_name)} ORDER BY version DESC LIMIT 1")
-        current_cols = current_json ? JSON.parse(current_json) : []
-
-        new_col = { "name" => column_name.to_s, "virgodb_type" => type.to_s, "nullable" => options[:null] != false }
-        insert_schema_version!(table_name, current_cols + [ new_col ])
-      end
-
-      def insert_schema_version!(table_name, cols)
-        next_version = (select_value("SELECT MAX(version) FROM schema_versions WHERE table_name = #{quote(table_name)}") || 0).to_i + 1
-
-        execute(<<~SQL)
-          INSERT INTO schema_versions (table_name, version, columns, applied_at)
-          VALUES (#{quote(table_name)}, #{next_version}, #{quote(cols.to_json)}, #{Time.now.to_i})
-        SQL
-      end
-
-      # Same DDL as Manifest::open()'s bootstrap on the Rust side (same
-      # table/column names, including the `table_name` scoping column) --
-      # IF NOT EXISTS makes this safe to call whether virgodb's own Rust
-      # code or this Ruby adapter created the manifest file first. Plain
-      # INTEGER PRIMARY KEY, not AUTOINCREMENT -- see the matching comment
-      # in manifest/src/lib.rs for why (breaks a real structure_dump/
-      # structure_load round trip the moment any host app's
-      # ActiveRecord::SchemaDumper.ignore_tables is non-empty).
-      def ensure_schema_versions_table!
-        execute(<<~SQL)
-          CREATE TABLE IF NOT EXISTS schema_versions (
-            id         INTEGER PRIMARY KEY,
-            table_name TEXT NOT NULL,
-            version    INTEGER NOT NULL,
-            columns    TEXT NOT NULL,
-            applied_at INTEGER NOT NULL
-          )
-        SQL
-      end
-    end
-  end
-end
-
-ActiveRecord::ConnectionAdapters.register(
-  "virgodb",
-  "ActiveRecord::ConnectionAdapters::VirgodbAdapter",
-  "activerecord-virgodb-adapter"
-)
-
-module ActiveRecord
-  module Tasks
-    # `ActiveRecord::Tasks::SQLiteDatabaseTasks#structure_dump` just shells
-    # out to the real `sqlite3` CLI (`.schema --nosys`, or a raw `SELECT sql
-    # FROM sqlite_master` when a host app has any
-    # `ActiveRecord::SchemaDumper.ignore_tables` configured -- e.g. excluding
-    # tables owned by a separate replication/backup mechanism) against the
-    # configured database file --
-    # DDL only. `ActiveRecord::Tasks::DatabaseTasks.dump_schema` then
-    # special-cases exactly one table on top of that: if `schema_migrations`
-    # exists, it appends that table's real row data (`connection
-    # .dump_schema_versions`) to the same dump file, because Rails' own
-    # migration bookkeeping needs those rows to survive a `db:schema:load`/
-    # `structure_load`, not just the table shape.
-    #
-    # `schema_versions` (this adapter's own bookkeeping table, plural,
-    # distinct from Rails' singular `schema_migrations`) needs exactly the
-    # same treatment and doesn't get it for free: it holds real data (the
-    # versioned column list `compact::schema_from_columns` on the Rust side
-    # depends on), not just structure, and Rails has no way to know that a
-    # table it didn't create needs the same row-preserving special case.
-    # Confirmed empirically: without this, a fresh database prepared via
-    # `structure_load` (which Rails does automatically whenever a `db:migrate`
-    # invocation for a later environment/connection finds an existing,
-    # version-matching dump already on disk from an earlier one -- not
-    # something this adapter can opt out of) ends up with the real
-    # host-app table but an EMPTY `schema_versions`, so
-    # `Manifest::current_schema_version()` on the Rust side returns `None`
-    # even though the SQLite table itself is correct.
-    class VirgodbDatabaseTasks < SQLiteDatabaseTasks
-      class << self
-        # Optional host-app hook: `->(sql_table_name) { {sort_by:, key_column:,
-        # partition_column:} or nil }`. Purely a documentation comment written
-        # above CREATE TABLE at dump time -- nothing on the Rust or Ruby side
-        # ever reads it back. This adapter gem stays host-app-agnostic (see
-        # the class doc comment above: "lets any Rails app..."), so it can't
-        # hardcode any one host app's sort_by/key_column config (that's
-        # app/services/virgodb/registry.rb's job, which already has this
-        # exact data for real -- passed to Virgodb.start_maintenance on every
-        # boot). A host app that wants it surfaced sets this once, e.g. in an
-        # initializer; left nil, structure_dump behaves exactly as before.
-        attr_accessor :table_layout_provider
-      end
-
-      def structure_dump(filename, extra_flags)
-        super
-        reformat_create_table_statements!(filename)
-        annotate_physical_layout!(filename)
-        dump_schema_versions!(filename)
-      end
-
-      private
-
-      # Only the LATEST schema_versions row per table_name is dumped.
-      # `Manifest::current_schema_version` (manifest/src/lib.rs) only ever
-      # reads `ORDER BY version DESC LIMIT 1` -- confirmed directly, no Rust
-      # or Ruby code anywhere reads an older version row, so keeping full
-      # history here isn't a correctness requirement. It also isn't a
-      # readability win: each version's `columns` re-lists every column from
-      # scratch, so N migrations means N near-duplicate JSON blobs, longer
-      # every time. The version count/dates are still worth one line for
-      # orientation -- the full column list per version isn't; the
-      # migration files themselves (db/migrate_virgodb_*/*.rb, timestamped,
-      # under git blame) are already the authoritative "what changed when"
-      # record. `register_schema_version`'s additive-only check is
-      # unaffected: it only ever compares a new migration's proposed columns
-      # against the CURRENT version, which is exactly what's still dumped.
-      def dump_schema_versions!(filename)
-        return unless connection.data_source_exists?("schema_versions")
-
-        rows = connection.select_all("SELECT id, table_name, version, columns, applied_at FROM schema_versions ORDER BY table_name, version")
-        return if rows.to_a.empty?
-
-        File.open(filename, "a") do |f|
-          f.puts
-          rows.to_a.group_by { |r| r["table_name"] }.each do |table_name, versions|
-            latest = versions.max_by { |r| r["version"] }
-            history = versions.map { |r| "v#{r['version']} (#{Time.at(r['applied_at'].to_i).utc.strftime('%Y-%m-%d')})" }.join(", ")
-
-            f.puts "-- #{table_name} schema history: #{history} -- only the current version is dumped below; see db/migrate_virgodb_*/ for the rest"
-            pretty_columns = JSON.pretty_generate(JSON.parse(latest["columns"]))
-            f.puts(
-              "INSERT INTO \"schema_versions\" (id, table_name, version, columns, applied_at) VALUES (\n" \
-              "  #{latest['id']}, #{connection.quote(table_name)}, #{latest['version']},\n" \
-              "  #{connection.quote(pretty_columns)},\n" \
-              "  #{latest['applied_at']}\n" \
-              ");"
-            )
-            f.puts
-          end
-        end
-      end
-
-      # Writes `table_layout_provider`'s sort_by/key_column/partition_column
-      # as a comment directly above each table's (already reformatted,
-      # one-column-per-line) CREATE TABLE -- the closest a SQLite-flavored
-      # dump can get to ClickHouse's own `ORDER BY`/`PARTITION BY` clauses
-      # being right there in the DDL. No-op (and no dump format change at
-      # all) when no provider is registered.
-      def annotate_physical_layout!(filename)
-        provider = self.class.table_layout_provider
-        return unless provider
-
-        content = File.read(filename)
-        content.gsub!(/^CREATE TABLE (IF NOT EXISTS )?"?(\w+)"? \(/) do
-          statement_start = Regexp.last_match(0)
-          table_name = Regexp.last_match(2)
-          layout = provider.call(table_name)
-          next statement_start unless layout
-
-          comment = [
-            "-- Physical layout (app/services/virgodb/registry.rb):",
-            "--   sort_by:          #{layout[:sort_by].join(', ')}",
-            "--   key_column:       #{layout[:key_column]} (run file min/max pruning bounds)",
-            "--   partition_column: #{layout[:partition_column]}"
-          ].join("\n")
-
-          "#{comment}\n#{statement_start}"
-        end
-        File.write(filename, content)
-      end
-
-      # `SQLite3Adapter#structure_dump`'s raw `SELECT sql FROM sqlite_master`
-      # path (used whenever `ActiveRecord::SchemaDumper.ignore_tables` is
-      # non-empty, e.g. excluding tables owned by a separate replication/
-      # backup mechanism) reproduces each
-      # CREATE TABLE exactly as ActiveRecord originally generated it --
-      # one long line, every column crammed together. Fine for SQLite to
-      # load back (structure_load doesn't care about whitespace), useless to
-      # a human reading the dump next to `db/clickhouse_structure.sql`
-      # (ClickHouse's own `SHOW CREATE TABLE` is naturally one-column-per-line,
-      # for comparison). Purely cosmetic, safe to do unconditionally: only
-      # rewrites the exact same statement with different whitespace, no DDL
-      # semantics change. Skips statements that already span multiple lines
-      # (e.g. this adapter's own hand-written `schema_versions` DDL, created
-      # via a real newline-containing heredoc) -- nothing to reformat there.
-      def reformat_create_table_statements!(filename)
-        content = File.read(filename)
-        content.gsub!(/^CREATE TABLE (IF NOT EXISTS )?("[\w]+"|\w+) \((.*)\);$/) do
-          if_not_exists = Regexp.last_match(1)
-          table_name = Regexp.last_match(2)
-          columns = split_top_level_commas(Regexp.last_match(3))
-          next Regexp.last_match(0) if columns.size <= 1
-
-          "CREATE TABLE #{if_not_exists}#{table_name} (\n" + columns.map { |c| "  #{c.strip}" }.join(",\n") + "\n);"
-        end
-        File.write(filename, content)
-      end
-
-      # Splits a CREATE TABLE column list on commas that are NOT nested
-      # inside a type's own parens (e.g. `Decimal(18,2)`) -- a plain
-      # `String#split(",")` would wrongly cut `Decimal(18` and `2)` apart.
-      def split_top_level_commas(str)
-        parts = []
-        depth = 0
-        current = +""
-        str.each_char do |ch|
-          case ch
-          when "(" then depth += 1; current << ch
-          when ")" then depth -= 1; current << ch
-          when ","
-            depth.zero? ? (parts << current; current = +"") : current << ch
-          else
-            current << ch
-          end
-        end
-        parts << current unless current.strip.empty?
-        parts
-      end
-    end
-  end
-end
-
-# Registering our own subclass (not plain SQLiteDatabaseTasks) so
-# schema_versions' row data survives the dump/load round trip -- see the
-# class doc comment above for why a real Rails app can't avoid hitting this.
-ActiveRecord::Tasks::DatabaseTasks.register_task(/virgodb/, "ActiveRecord::Tasks::VirgodbDatabaseTasks")
+# exactly) so Bundler's default require (`require gem_name`, no `require:`
+# option needed in the Gemfile) actually finds it. The real code lives at
+# the conventional-looking paths below (mirroring how Rails organizes its
+# own built-in adapters) -- this file is just Bundler's entrypoint into
+# them. Rails' own ActiveRecord::ConnectionAdapters.register/
+# DatabaseTasks.register_task (bottom of each file below) treat the require
+# path passed to them as an opaque string, not a real convention, so
+# pointing them at this file rather than either individual one is what
+# makes their lazy-load fallback path (loading before Bundler's own
+# implicit require has run) reach both classes, not just one.
+require_relative "active_record/connection_adapters/virgodb_adapter"
+require_relative "active_record/tasks/virgodb_database_tasks"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-virgodb-adapter
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - virgodb
@@ -23,6 +23,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -39,7 +53,7 @@ dependencies:
         version: '1.4'
 description: |
   Lets any Rails app write real ActiveRecord::Migration subclasses against a virgodb manifest,
-  picked up by `rails db:migrate` the same way SQLite and ClickHouse migrations already are.
+  picked up by `rails db:migrate` the same way SQLite migrations already are.
   Subclasses ActiveRecord::ConnectionAdapters::SQLite3Adapter (virgodb's manifest is already a
   real SQLite file), so structure_dump/the :sql schema format work for free.
 executables: []
@@ -47,6 +61,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
+- lib/active_record/connection_adapters/virgodb_adapter.rb
+- lib/active_record/tasks/virgodb_database_tasks.rb
 - lib/activerecord-virgodb-adapter.rb
 licenses:
 - Nonstandard