activerecord-virgodb-adapter 0.1.0.dev.1424cbd → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2648b136e688ad91ca03228e433bf7f7fa4665fb78d76a3dfc39a0753dc24094
-  data.tar.gz: d046ad46492f2de29816497f566095f77d48defe0343f1bf6c44c9a455edc0bf
+  metadata.gz: 757fecfe8176d3354cf42df451f013c814c2699b3ac1206a055a3ada8196fd10
+  data.tar.gz: 945f9ab23d59f055951dcbe55786545f106d7c0c90a26c23699a9e61a32cbec2
 SHA512:
-  metadata.gz: b4c229b5f97009e41ee6a795bd2308f13d697438eb16a846fafcf21aa252e1447801b09562ec79902bf387de184c6044368a27a9dc4e1270438d77ad2dee35c8
-  data.tar.gz: fd8ff15112deb4bae56553011e7f2ac6e5d73a3d8847b276fdda719e4480a1f5a8d4b691c0316ee1056aa6d292ca3f9fa316f12fb046ec2423ae7927a8946d7a
+  metadata.gz: 24f8356243e4ab3349e1b94c4fb4d561d28e86575442bca4e7185cb1c2a39c745c3f98a25b7bdca0e06f09b5f9cf50af121963fba72d021d2ab3e61b53461dc3
+  data.tar.gz: f9c6ced6922d7254082c043e2317507d303b63fb4708a957ca8cf33f6fb462711e72690a19074639f37138f5d0e80c884ec13cc0e08d4fd9255edc99ad0df026

data/lib/active_record/connection_adapters/virgodb_adapter.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
 require "active_record/connection_adapters/sqlite3_adapter"
+require "bigdecimal"
 require "json"
 
 module ActiveRecord
   module ConnectionAdapters
     # ActiveRecord adapter for virgodb migrations -- lets any Rails app
     # define and evolve a virgodb table's schema the same way it already
-    # does for SQLite (bin/rails db:migrate) and ClickHouse (the
-    # clickhouse-activerecord gem): real ActiveRecord::Migration subclasses,
-    # raw type strings, picked up by `rails db:migrate` alongside every
-    # other database.
+    # does for SQLite (bin/rails db:migrate): real ActiveRecord::Migration
+    # subclasses, raw type strings, picked up by `rails db:migrate`
+    # alongside every other database.
     #
     # virgodb's manifest is already a real SQLite file, so subclassing
     # SQLite3Adapter (not building from AbstractAdapter) gets
@@ -40,6 +40,59 @@ module ActiveRecord
     # `rename_column`, and `change_column` are all disabled outright rather
     # than silently letting the real SQLite table drift out of sync with
     # what `schema_versions` describes.
+
+    # virgodb's raw "Timestamp" type (crates/compact/src/schema.rs) is plain
+    # UNIX seconds -- an Integer, not a real SQL DATETIME/TIMESTAMP string --
+    # but it still matches Rails' generic %r(time)i registration (see
+    # VirgodbAdapter.initialize_type_map below), landing on the stock
+    # ActiveRecord::Type::DateTime, which only knows how to parse a real
+    # date/time STRING (or an already-real Time/Date/Hash). Two confirmed,
+    # real failure modes from that mismatch, not just cosmetic ones:
+    #
+    # 1. `Type::DateTime.new.cast("0")` -- "0" being a perfectly valid
+    #    epoch-seconds default (e.g. exit_page_at/split_test_id_at's own
+    #    DEFAULT 0 sentinel, see
+    #    db/migrate_virgodb/20260727000002_create_ahoy_visits.rb in hintpot)
+    #    -- returns nil. `Column#default` itself is unaffected (DateTime is
+    #    `mutable?`, so `Column#initialize` keeps the raw string rather than
+    #    deserializing it -- see ActiveModel::Type::DateTime#mutable?'s own
+    #    comment), which is why the schema_versions capture above was never
+    #    wrong. But `Model.column_defaults` (what a real host app's own
+    #    schema-annotation tooling reads) DOES deserialize through the cast
+    #    type, surfacing as a misleading `default(NULL)` in a real model's
+    #    annotation for a column that has a perfectly real default.
+    # 2. `Type::DateTime.new.cast(1753000000)` (a real Integer write-time
+    #    value) ALSO doesn't produce a real Time object -- DateTime#cast
+    #    only recognizes String/Time/Date/Hash input, so an Integer just
+    #    passes through unchanged. Harmless today (nothing in this
+    #    codebase reads a Timestamp column back through a real
+    #    ActiveRecord attribute -- see Virgodb::Registry.query, which
+    #    returns raw parsed JSON, bypassing AR type casting entirely), but
+    #    a real latent gap the moment anything ever does.
+    #
+    # Subclassing ActiveRecord::Type::DateTime (not building from scratch)
+    # keeps `.type == :datetime` -- the semantically correct label for a
+    # timestamp column, unlike the columns String fix above where Rails had
+    # no keyword at all -- and keeps every other DateTime behavior (real
+    # date strings, Time/Date input, timezone handling) working exactly as
+    # before; only the epoch-seconds-as-Integer-or-String case needed a
+    # fix. `deserialize` isn't overridden separately -- ActiveModel::Type
+    # ::Value#deserialize just calls `cast` by default (confirmed: neither
+    # ActiveModel::Type::DateTime nor ActiveRecord::Type::DateTime override
+    # it), so fixing `cast` alone is enough for both directions.
+    class VirgodbTimestamp < ActiveRecord::Type::DateTime
+      def cast(value)
+        return value if value.nil? || value.is_a?(::Time) || value.is_a?(::DateTime)
+
+        seconds = Float(value, exception: false)
+        seconds ? ::Time.at(seconds).utc : super
+      end
+
+      def serialize(value)
+        value.respond_to?(:to_time) ? value.to_time.to_i : value
+      end
+    end
+
     class VirgodbAdapter < SQLite3Adapter
       ADAPTER_NAME = "Virgodb"
 
@@ -57,11 +110,105 @@ module ActiveRecord
       # issued immediately after `establish_connection`, before ANY table or
       # migration ran, still silently failed. The pragma has to be set
       # before `super` runs here, not merely before the first CREATE TABLE.
+      #
+      # Second real bug, found the same way: `configure_connection` runs on
+      # EVERY new raw connection this adapter's pool ever opens, not just
+      # the manifest's first one -- so an unconditional `auto_vacuum =`
+      # unconditionally re-issued `PRAGMA auto_vacuum='2'` on every single
+      # one of them, forever. Once a manifest is genuinely in INCREMENTAL
+      # mode (the common case for any real, already-migrated manifest),
+      # re-setting it to that SAME value is not the free no-op re-setting an
+      # already-NONE manifest is: confirmed directly with a raw SQLite3
+      # connection, re-issuing the SET pragma while a second connection
+      # holds an open `BEGIN IMMEDIATE` write transaction (exactly what
+      # virgodb's own maintenance thread does while compacting, see
+      # crates/compact/src/maintenance.rs) blocks for the full busy_timeout
+      # and then raises SQLite3::BusyException -- surfacing in a host app as
+      # an ActiveRecord::StatementTimeout on whatever request happened to
+      # need a brand-new pooled connection at that moment. A bare read
+      # (`PRAGMA auto_vacuum` with no argument) does NOT block the same way,
+      # confirmed the same way -- so skip the write once it's already
+      # reached the target value instead of paying that contention risk on
+      # every connection after the first.
       def configure_connection
-        @raw_connection.auto_vacuum = "incremental" if @raw_connection.respond_to?(:auto_vacuum=)
+        if @raw_connection.respond_to?(:auto_vacuum=) && @raw_connection.auto_vacuum != 2
+          @raw_connection.auto_vacuum = "incremental"
+        end
         super
       end
 
+      class << self
+        private
+
+        # Real bug, found (not assumed) by a host app trying to build real
+        # ActiveRecord models on top of virgodb tables: `Column#type` (what
+        # every host app actually reads -- annotation generators, `cast`,
+        # attribute serialization) comes from `TYPE_MAP.lookup(sql_type)`,
+        # and Rails' own %r(char)i / %r(int)i / %r(date)i / etc. regexes
+        # (registered by AbstractAdapter#initialize_type_map, SQLite3Adapter
+        # only adds `int`) exist to recognize real SQL DDL keywords like
+        # "varchar" or "datetime" -- never anything from virgodb's own raw
+        # type vocabulary (crates/compact/src/schema.rs), since `t.column
+        # name, "raw string"` bypasses Rails' normal :string/:datetime/etc
+        # -> DDL-keyword translation entirely. Two confirmed failure modes:
+        #
+        # 1. A bare "String" or "LowCardinality String" column matches NONE
+        #    of Rails' base regexes (there's no keyword for it -- real
+        #    SQLite migrations only ever produce "varchar"), so `.type`
+        #    falls through to `Type.default_value`, an untyped
+        #    `ActiveModel::Type::Value` whose `.type` is nil.
+        # 2. An `AnyLast <type> <order_by_column>` column (the two-scalar-
+        #    column tuple-workaround pairs -- see
+        #    db/migrate_virgodb/20260727000002_create_ahoy_visits.rb in
+        #    hintpot) can get a WRONG match instead of no match: Rails'
+        #    %r(date)i is an unanchored substring regex, and
+        #    "AnyLast String updated_at" contains the substring "date"
+        #    (from "upDATEd_at") purely by accident, mis-casting a String
+        #    column as :date. Confirmed directly: TYPE_MAP.lookup("AnyLast
+        #    String updated_at").class == ActiveRecord::Type::Date.
+        #
+        # The fix strips virgodb's own wrapper syntax down to the bare base
+        # type and re-enters `m.lookup` for it, rather than trying to write
+        # an ever-more-specific whole-string regex per aggregate function x
+        # base type combination -- that would still be one accidental
+        # trailing-column-name substring match away from the same bug the
+        # moment a future order-by column happens to contain "int"/"char"/
+        # "time"/etc. `AnyLast` is the only aggregate function that ever
+        # carries a trailing order-by column (it's the one that needs to
+        # know which row is "last"; Any/Sum/Max/Min/UniqueArrayUnion are
+        # simple scalar aggregates with no ordering to track), so it's the
+        # only one that needs the extra trailing-token strip.
+        def initialize_type_map(m)
+          super
+
+          register_class_with_limit m, /\A(?:LowCardinality\s+)?String\z/i, Type::String
+          m.register_type(/\ATimestamp\z/i, VirgodbTimestamp.new)
+
+          m.register_type(/\AAnyLast\s+/i) do |sql_type|
+            base = sql_type.sub(/\AAnyLast\s+/i, "").sub(/\s+\S+\z/, "")
+            m.lookup(base)
+          end
+
+          m.register_type(/\A(?:Any|Sum|Max|Min|UniqueArrayUnion)\s+/i) do |sql_type|
+            base = sql_type.sub(/\A(?:Any|Sum|Max|Min|UniqueArrayUnion)\s+/i, "")
+            m.lookup(base)
+          end
+        end
+      end
+
+      # `type_map` (AbstractAdapter) reads `self.class::TYPE_MAP` -- a plain
+      # constant lookup, not a method call -- so merely overriding
+      # `initialize_type_map` above does nothing on its own: without a
+      # `TYPE_MAP` constant of ITS OWN, `VirgodbAdapter::TYPE_MAP` constant
+      # lookup just walks up to the already-frozen `SQLite3Adapter::TYPE_MAP`
+      # (built at THAT class's load time, from the un-overridden method).
+      # Confirmed directly: the override above alone left every virgodb
+      # column's `.type` unchanged. SQLite3Adapter builds its own `TYPE_MAP`
+      # constant the exact same way (see its `TYPE_MAP = Type::TypeMap.new
+      # .tap { |m| initialize_type_map(m) }` line) specifically so subclasses
+      # can shadow it like this.
+      TYPE_MAP = Type::TypeMap.new.tap { |m| initialize_type_map(m) }
+
       def create_table(table_name, **options, &block)
         result = super
         record_schema_version!(table_name)
@@ -124,12 +271,34 @@ module ActiveRecord
         ensure_schema_versions_table!
 
         cols = columns(table_name).map do |c|
-          { "name" => c.name, "virgodb_type" => c.sql_type, "nullable" => c.null }
+          { "name" => c.name, "virgodb_type" => c.sql_type, "nullable" => c.null, "default" => stringify_default(c.default) }
         end
 
         insert_schema_version!(table_name, cols)
       end
 
+      # A migration's `default:` (e.g. `t.column :clid, "String", null:
+      # false, default: ""`) round-trips through `columns(table_name).first
+      # .default` as a real, correctly-typed Ruby value now (String/Integer/
+      # BigDecimal) -- only possible since the TYPE_MAP fix above; before it,
+      # every virgodb String column's `.type` was nil/wrong and `.default`
+      # came back nil regardless of what the migration declared. schema_versions
+      # itself only stores plain JSON strings (matching `virgodb_type`'s own
+      # convention of storing raw text the Rust side parses, not a typed
+      # value) -- the Rust write path (crates/virgodb_ruby's row_convert.rs)
+      # re-parses this string against the column's real Arrow type at write
+      # time. `BigDecimal#to_s` explicitly forced to fixed-point ("F") rather
+      # than its default format, which switches to scientific notation for
+      # some magnitudes -- avoids writing e.g. "0.12345e3" into
+      # schema_versions where a human reading a real dump would expect
+      # "123.45" (Rust's f64 parser would actually accept either form, this
+      # is purely for a human reading a real structure.sql/schema_versions
+      # dump, not a correctness requirement).
+      def stringify_default(value)
+        return nil if value.nil?
+        value.is_a?(BigDecimal) ? value.to_s("F") : value.to_s
+      end
+
       # `add_column`'s schema_versions update -- deliberately does NOT
       # re-introspect the whole table via `columns(table_name)` the way
       # `record_schema_version!` does. Real bug, found (not assumed) by
@@ -162,7 +331,7 @@ module ActiveRecord
         current_json = select_value("SELECT columns FROM schema_versions WHERE table_name = #{quote(table_name)} ORDER BY version DESC LIMIT 1")
         current_cols = current_json ? JSON.parse(current_json) : []
 
-        new_col = { "name" => column_name.to_s, "virgodb_type" => type.to_s, "nullable" => options[:null] != false }
+        new_col = { "name" => column_name.to_s, "virgodb_type" => type.to_s, "nullable" => options[:null] != false, "default" => stringify_default(options[:default]) }
         insert_schema_version!(table_name, current_cols + [ new_col ])
       end
 
@@ -201,183 +370,5 @@ end
 ActiveRecord::ConnectionAdapters.register(
   "virgodb",
   "ActiveRecord::ConnectionAdapters::VirgodbAdapter",
-  "active_record/connection_adapters/virgodb_adapter"
+  "activerecord-virgodb-adapter"
 )
-
-module ActiveRecord
-  module Tasks
-    # `ActiveRecord::Tasks::SQLiteDatabaseTasks#structure_dump` just shells
-    # out to the real `sqlite3` CLI (`.schema --nosys`, or a raw `SELECT sql
-    # FROM sqlite_master` when a host app has any
-    # `ActiveRecord::SchemaDumper.ignore_tables` configured -- e.g. excluding
-    # tables owned by a separate replication/backup mechanism) against the
-    # configured database file --
-    # DDL only. `ActiveRecord::Tasks::DatabaseTasks.dump_schema` then
-    # special-cases exactly one table on top of that: if `schema_migrations`
-    # exists, it appends that table's real row data (`connection
-    # .dump_schema_versions`) to the same dump file, because Rails' own
-    # migration bookkeeping needs those rows to survive a `db:schema:load`/
-    # `structure_load`, not just the table shape.
-    #
-    # `schema_versions` (this adapter's own bookkeeping table, plural,
-    # distinct from Rails' singular `schema_migrations`) needs exactly the
-    # same treatment and doesn't get it for free: it holds real data (the
-    # versioned column list `compact::schema_from_columns` on the Rust side
-    # depends on), not just structure, and Rails has no way to know that a
-    # table it didn't create needs the same row-preserving special case.
-    # Confirmed empirically: without this, a fresh database prepared via
-    # `structure_load` (which Rails does automatically whenever a `db:migrate`
-    # invocation for a later environment/connection finds an existing,
-    # version-matching dump already on disk from an earlier one -- not
-    # something this adapter can opt out of) ends up with the real
-    # host-app table but an EMPTY `schema_versions`, so
-    # `Manifest::current_schema_version()` on the Rust side returns `None`
-    # even though the SQLite table itself is correct.
-    class VirgodbDatabaseTasks < SQLiteDatabaseTasks
-      class << self
-        # Optional host-app hook: `->(sql_table_name) { {sort_by:, key_column:,
-        # partition_column:} or nil }`. Purely a documentation comment written
-        # above CREATE TABLE at dump time -- nothing on the Rust or Ruby side
-        # ever reads it back. This adapter gem stays host-app-agnostic (see
-        # the class doc comment above: "lets any Rails app..."), so it can't
-        # hardcode any one host app's sort_by/key_column config (that's
-        # app/services/virgodb/registry.rb's job, which already has this
-        # exact data for real -- passed to Virgodb.start_maintenance on every
-        # boot). A host app that wants it surfaced sets this once, e.g. in an
-        # initializer; left nil, structure_dump behaves exactly as before.
-        attr_accessor :table_layout_provider
-      end
-
-      def structure_dump(filename, extra_flags)
-        super
-        reformat_create_table_statements!(filename)
-        annotate_physical_layout!(filename)
-        dump_schema_versions!(filename)
-      end
-
-      private
-
-      # Only the LATEST schema_versions row per table_name is dumped.
-      # `Manifest::current_schema_version` (manifest/src/lib.rs) only ever
-      # reads `ORDER BY version DESC LIMIT 1` -- confirmed directly, no Rust
-      # or Ruby code anywhere reads an older version row, so keeping full
-      # history here isn't a correctness requirement. It also isn't a
-      # readability win: each version's `columns` re-lists every column from
-      # scratch, so N migrations means N near-duplicate JSON blobs, longer
-      # every time. The version count/dates are still worth one line for
-      # orientation -- the full column list per version isn't; the
-      # migration files themselves (db/migrate_virgodb_*/*.rb, timestamped,
-      # under git blame) are already the authoritative "what changed when"
-      # record. `register_schema_version`'s additive-only check is
-      # unaffected: it only ever compares a new migration's proposed columns
-      # against the CURRENT version, which is exactly what's still dumped.
-      def dump_schema_versions!(filename)
-        return unless connection.data_source_exists?("schema_versions")
-
-        rows = connection.select_all("SELECT id, table_name, version, columns, applied_at FROM schema_versions ORDER BY table_name, version")
-        return if rows.to_a.empty?
-
-        File.open(filename, "a") do |f|
-          f.puts
-          rows.to_a.group_by { |r| r["table_name"] }.each do |table_name, versions|
-            latest = versions.max_by { |r| r["version"] }
-            history = versions.map { |r| "v#{r['version']} (#{Time.at(r['applied_at'].to_i).utc.strftime('%Y-%m-%d')})" }.join(", ")
-
-            f.puts "-- #{table_name} schema history: #{history} -- only the current version is dumped below; see db/migrate_virgodb_*/ for the rest"
-            pretty_columns = JSON.pretty_generate(JSON.parse(latest["columns"]))
-            f.puts(
-              "INSERT INTO \"schema_versions\" (id, table_name, version, columns, applied_at) VALUES (\n" \
-              "  #{latest['id']}, #{connection.quote(table_name)}, #{latest['version']},\n" \
-              "  #{connection.quote(pretty_columns)},\n" \
-              "  #{latest['applied_at']}\n" \
-              ");"
-            )
-            f.puts
-          end
-        end
-      end
-
-      # Writes `table_layout_provider`'s sort_by/key_column/partition_column
-      # as a comment directly above each table's (already reformatted,
-      # one-column-per-line) CREATE TABLE -- the closest a SQLite-flavored
-      # dump can get to ClickHouse's own `ORDER BY`/`PARTITION BY` clauses
-      # being right there in the DDL. No-op (and no dump format change at
-      # all) when no provider is registered.
-      def annotate_physical_layout!(filename)
-        provider = self.class.table_layout_provider
-        return unless provider
-
-        content = File.read(filename)
-        content.gsub!(/^CREATE TABLE (IF NOT EXISTS )?"?(\w+)"? \(/) do
-          statement_start = Regexp.last_match(0)
-          table_name = Regexp.last_match(2)
-          layout = provider.call(table_name)
-          next statement_start unless layout
-
-          comment = [
-            "-- Physical layout (app/services/virgodb/registry.rb):",
-            "--   sort_by:          #{layout[:sort_by].join(', ')}",
-            "--   key_column:       #{layout[:key_column]} (run file min/max pruning bounds)",
-            "--   partition_column: #{layout[:partition_column]}"
-          ].join("\n")
-
-          "#{comment}\n#{statement_start}"
-        end
-        File.write(filename, content)
-      end
-
-      # `SQLite3Adapter#structure_dump`'s raw `SELECT sql FROM sqlite_master`
-      # path (used whenever `ActiveRecord::SchemaDumper.ignore_tables` is
-      # non-empty, e.g. excluding tables owned by a separate replication/
-      # backup mechanism) reproduces each
-      # CREATE TABLE exactly as ActiveRecord originally generated it --
-      # one long line, every column crammed together. Fine for SQLite to
-      # load back (structure_load doesn't care about whitespace), useless to
-      # a human reading the dump next to `db/clickhouse_structure.sql`
-      # (ClickHouse's own `SHOW CREATE TABLE` is naturally one-column-per-line,
-      # for comparison). Purely cosmetic, safe to do unconditionally: only
-      # rewrites the exact same statement with different whitespace, no DDL
-      # semantics change. Skips statements that already span multiple lines
-      # (e.g. this adapter's own hand-written `schema_versions` DDL, created
-      # via a real newline-containing heredoc) -- nothing to reformat there.
-      def reformat_create_table_statements!(filename)
-        content = File.read(filename)
-        content.gsub!(/^CREATE TABLE (IF NOT EXISTS )?("[\w]+"|\w+) \((.*)\);$/) do
-          if_not_exists = Regexp.last_match(1)
-          table_name = Regexp.last_match(2)
-          columns = split_top_level_commas(Regexp.last_match(3))
-          next Regexp.last_match(0) if columns.size <= 1
-
-          "CREATE TABLE #{if_not_exists}#{table_name} (\n" + columns.map { |c| "  #{c.strip}" }.join(",\n") + "\n);"
-        end
-        File.write(filename, content)
-      end
-
-      # Splits a CREATE TABLE column list on commas that are NOT nested
-      # inside a type's own parens (e.g. `Decimal(18,2)`) -- a plain
-      # `String#split(",")` would wrongly cut `Decimal(18` and `2)` apart.
-      def split_top_level_commas(str)
-        parts = []
-        depth = 0
-        current = +""
-        str.each_char do |ch|
-          case ch
-          when "(" then depth += 1; current << ch
-          when ")" then depth -= 1; current << ch
-          when ","
-            depth.zero? ? (parts << current; current = +"") : current << ch
-          else
-            current << ch
-          end
-        end
-        parts << current unless current.strip.empty?
-        parts
-      end
-    end
-  end
-end
-
-# Registering our own subclass (not plain SQLiteDatabaseTasks) so
-# schema_versions' row data survives the dump/load round trip -- see the
-# class doc comment above for why a real Rails app can't avoid hitting this.
-ActiveRecord::Tasks::DatabaseTasks.register_task(/virgodb/, "ActiveRecord::Tasks::VirgodbDatabaseTasks")

data/lib/active_record/tasks/virgodb_database_tasks.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+require "active_record/tasks/sqlite_database_tasks"
+require "json"
+require "sqlite3"
+
+module ActiveRecord
+  module Tasks
+    # `ActiveRecord::Tasks::SQLiteDatabaseTasks#structure_dump` just shells
+    # out to the real `sqlite3` CLI (`.schema --nosys`, or a raw `SELECT sql
+    # FROM sqlite_master` when a host app has any
+    # `ActiveRecord::SchemaDumper.ignore_tables` configured -- e.g. excluding
+    # tables owned by a separate replication/backup mechanism) against the
+    # configured database file --
+    # DDL only. `ActiveRecord::Tasks::DatabaseTasks.dump_schema` then
+    # special-cases exactly one table on top of that: if `schema_migrations`
+    # exists, it appends that table's real row data (`connection
+    # .dump_schema_versions`) to the same dump file, because Rails' own
+    # migration bookkeeping needs those rows to survive a `db:schema:load`/
+    # `structure_load`, not just the table shape.
+    #
+    # `schema_versions` (this adapter's own bookkeeping table, plural,
+    # distinct from Rails' singular `schema_migrations`) needs exactly the
+    # same treatment and doesn't get it for free: it holds real data (the
+    # versioned column list `compact::schema_from_columns` on the Rust side
+    # depends on), not just structure, and Rails has no way to know that a
+    # table it didn't create needs the same row-preserving special case.
+    # Confirmed empirically: without this, a fresh database prepared via
+    # `structure_load` (which Rails does automatically whenever a `db:migrate`
+    # invocation for a later environment/connection finds an existing,
+    # version-matching dump already on disk from an earlier one -- not
+    # something this adapter can opt out of) ends up with the real
+    # host-app table but an EMPTY `schema_versions`, so
+    # `Manifest::current_schema_version()` on the Rust side returns `None`
+    # even though the SQLite table itself is correct.
+    class VirgodbDatabaseTasks < SQLiteDatabaseTasks
+      class << self
+        # Optional host-app hook: `->(sql_table_name) { {sort_by:, key_column:,
+        # partition_column:} or nil }`. Purely a documentation comment written
+        # above CREATE TABLE at dump time -- nothing on the Rust or Ruby side
+        # ever reads it back. This adapter gem stays host-app-agnostic (see
+        # the class doc comment above: "lets any Rails app..."), so it can't
+        # hardcode any one host app's sort_by/key_column config (that's
+        # app/services/virgodb/registry.rb's job, which already has this
+        # exact data for real -- passed to Virgodb.start_maintenance on every
+        # boot). A host app that wants it surfaced sets this once, e.g. in an
+        # initializer; left nil, structure_dump behaves exactly as before.
+        attr_accessor :table_layout_provider
+
+        # Optional host-app hook: `->(sql_table_name) { path_or_nil }`.
+        # Convention over configuration, not opt-in like table_layout_provider
+        # above: this gem is Rails-specific (see the class doc comment
+        # above -- "Lets any Rails app..."), so it bakes in the same
+        # convention Rails' own generators use for SQLite's storage location
+        # (config/database.yml's generated `storage/#{Rails.env}.sqlite3`)
+        # -- `storage/#{Rails.env}_virgodb_#{sql_table_name}_data`, right
+        # next to the manifest. A host app gets working data-dir cleanup
+        # with zero setup. One with its own directory convention calls
+        # `data_dir_provider =` to override; `= nil` opts every table out
+        # entirely, same as plain SQLiteDatabaseTasks -- distinguished from
+        # "never touched this accessor" via `defined?` below so an explicit
+        # opt-out can't be silently overridden by the default.
+        def data_dir_provider
+          defined?(@data_dir_provider) ? @data_dir_provider : DEFAULT_DATA_DIR_PROVIDER
+        end
+        attr_writer :data_dir_provider
+      end
+
+      DEFAULT_DATA_DIR_PROVIDER = lambda do |sql_table_name|
+        Rails.root.join("storage", "#{Rails.env}_virgodb_#{sql_table_name}_data").to_s
+      end
+
+      def structure_dump(filename, extra_flags)
+        super
+        reformat_create_table_statements!(filename)
+        annotate_physical_layout!(filename)
+        dump_schema_versions!(filename)
+      end
+
+      # Overridden (not just inherited from SQLiteDatabaseTasks) so dropping
+      # the manifest also cleans up whatever data_dir_provider says each of
+      # its tables' physical files live under. db:drop only ever knew about
+      # the ONE file database.yml declares -- left alone, it silently
+      # orphans every Parquet run file and Tier 1 inline-batch pointer file
+      # forever, since there's no catalog left afterward to even reference
+      # them (compact::vacuum, which cleans those up via the manifest's own
+      # orphaned_files table, can't reach them either once the manifest's
+      # gone).
+      #
+      # Reads which tables exist via a fresh, read-only, raw SQLite3
+      # connection (not `connection`/`ActiveRecord::Base.lease_connection`
+      # -- unlike structure_dump, drop doesn't establish one for this
+      # specific db_config, and this shouldn't depend on whatever happens
+      # to be currently leased) BEFORE calling super, since super deletes
+      # the manifest file -- schema_versions goes with it.
+      def drop
+        provider = self.class.data_dir_provider
+        table_names = provider ? table_names_for_data_dir_cleanup : []
+        super
+        table_names.each do |table_name|
+          dir = provider.call(table_name)
+          FileUtils.rm_rf(dir) if dir
+        end
+      end
+
+      private
+
+      def table_names_for_data_dir_cleanup
+        db_path = db_config.database
+        path = File.absolute_path?(db_path) ? db_path : File.join(root, db_path)
+        return [] unless File.exist?(path)
+
+        raw = SQLite3::Database.new(path, readonly: true)
+        raw.busy_timeout = 5_000
+        raw.execute("SELECT DISTINCT table_name FROM schema_versions").flatten
+      ensure
+        raw&.close
+      end
+
+      # Only the LATEST schema_versions row per table_name is dumped.
+      # `Manifest::current_schema_version` (manifest/src/lib.rs) only ever
+      # reads `ORDER BY version DESC LIMIT 1` -- confirmed directly, no Rust
+      # or Ruby code anywhere reads an older version row, so keeping full
+      # history here isn't a correctness requirement. It also isn't a
+      # readability win: each version's `columns` re-lists every column from
+      # scratch, so N migrations means N near-duplicate JSON blobs, longer
+      # every time. The version count/dates are still worth one line for
+      # orientation -- the full column list per version isn't; the
+      # migration files themselves (db/migrate_virgodb_*/*.rb, timestamped,
+      # under git blame) are already the authoritative "what changed when"
+      # record. `register_schema_version`'s additive-only check is
+      # unaffected: it only ever compares a new migration's proposed columns
+      # against the CURRENT version, which is exactly what's still dumped.
+      def dump_schema_versions!(filename)
+        return unless connection.data_source_exists?("schema_versions")
+
+        rows = connection.select_all("SELECT id, table_name, version, columns, applied_at FROM schema_versions ORDER BY table_name, version")
+        return if rows.to_a.empty?
+
+        File.open(filename, "a") do |f|
+          f.puts
+          rows.to_a.group_by { |r| r["table_name"] }.each do |table_name, versions|
+            latest = versions.max_by { |r| r["version"] }
+            history = versions.map { |r| "v#{r['version']} (#{Time.at(r['applied_at'].to_i).utc.strftime('%Y-%m-%d')})" }.join(", ")
+
+            f.puts "-- #{table_name} schema history: #{history} -- only the current version is dumped below; see db/migrate_virgodb_*/ for the rest"
+            pretty_columns = JSON.pretty_generate(JSON.parse(latest["columns"]))
+            f.puts(
+              "INSERT INTO \"schema_versions\" (id, table_name, version, columns, applied_at) VALUES (\n" \
+              "  #{latest['id']}, #{connection.quote(table_name)}, #{latest['version']},\n" \
+              "  #{connection.quote(pretty_columns)},\n" \
+              "  #{latest['applied_at']}\n" \
+              ");"
+            )
+            f.puts
+          end
+        end
+      end
+
+      # Writes `table_layout_provider`'s sort_by/key_column/partition_column
+      # as a comment directly above each table's (already reformatted,
+      # one-column-per-line) CREATE TABLE -- the closest a SQLite-flavored
+      # dump can get to ClickHouse's own `ORDER BY`/`PARTITION BY` clauses
+      # being right there in the DDL. No-op (and no dump format change at
+      # all) when no provider is registered.
+      def annotate_physical_layout!(filename)
+        provider = self.class.table_layout_provider
+        return unless provider
+
+        content = File.read(filename)
+        content.gsub!(/^CREATE TABLE (IF NOT EXISTS )?"?(\w+)"? \(/) do
+          statement_start = Regexp.last_match(0)
+          table_name = Regexp.last_match(2)
+          layout = provider.call(table_name)
+          next statement_start unless layout
+
+          comment = [
+            "-- Physical layout (app/services/virgodb/registry.rb):",
+            "--   sort_by:          #{layout[:sort_by].join(', ')}",
+            "--   key_column:       #{layout[:key_column]} (run file min/max pruning bounds)",
+            "--   partition_column: #{layout[:partition_column]}"
+          ].join("\n")
+
+          "#{comment}\n#{statement_start}"
+        end
+        File.write(filename, content)
+      end
+
+      # `SQLite3Adapter#structure_dump`'s raw `SELECT sql FROM sqlite_master`
+      # path (used whenever `ActiveRecord::SchemaDumper.ignore_tables` is
+      # non-empty, e.g. excluding tables owned by a separate replication/
+      # backup mechanism) reproduces each
+      # CREATE TABLE exactly as ActiveRecord originally generated it --
+      # one long line, every column crammed together. Fine for SQLite to
+      # load back (structure_load doesn't care about whitespace), useless to
+      # a human reading the dump next to `db/clickhouse_structure.sql`
+      # (ClickHouse's own `SHOW CREATE TABLE` is naturally one-column-per-line,
+      # for comparison). Purely cosmetic, safe to do unconditionally: only
+      # rewrites the exact same statement with different whitespace, no DDL
+      # semantics change. Skips statements that already span multiple lines
+      # (e.g. this adapter's own hand-written `schema_versions` DDL, created
+      # via a real newline-containing heredoc) -- nothing to reformat there.
+      def reformat_create_table_statements!(filename)
+        content = File.read(filename)
+        content.gsub!(/^CREATE TABLE (IF NOT EXISTS )?("[\w]+"|\w+) \((.*)\);$/) do
+          if_not_exists = Regexp.last_match(1)
+          table_name = Regexp.last_match(2)
+          columns = split_top_level_commas(Regexp.last_match(3))
+          next Regexp.last_match(0) if columns.size <= 1
+
+          "CREATE TABLE #{if_not_exists}#{table_name} (\n" + columns.map { |c| "  #{c.strip}" }.join(",\n") + "\n);"
+        end
+        File.write(filename, content)
+      end
+
+      # Splits a CREATE TABLE column list on commas that are NOT nested
+      # inside a type's own parens (e.g. `Decimal(18,2)`) -- a plain
+      # `String#split(",")` would wrongly cut `Decimal(18` and `2)` apart.
+      def split_top_level_commas(str)
+        parts = []
+        depth = 0
+        current = +""
+        str.each_char do |ch|
+          case ch
+          when "(" then depth += 1; current << ch
+          when ")" then depth -= 1; current << ch
+          when ","
+            depth.zero? ? (parts << current; current = +"") : current << ch
+          else
+            current << ch
+          end
+        end
+        parts << current unless current.strip.empty?
+        parts
+      end
+    end
+  end
+end
+
+# Registering our own subclass (not plain SQLiteDatabaseTasks) so
+# schema_versions' row data survives the dump/load round trip -- see the
+# class doc comment above for why a real Rails app can't avoid hitting this.
+ActiveRecord::Tasks::DatabaseTasks.register_task(/virgodb/, "ActiveRecord::Tasks::VirgodbDatabaseTasks")

data/lib/activerecord-virgodb-adapter.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Lives at lib/activerecord-virgodb-adapter.rb (matching the gem name
+# exactly) so Bundler's default require (`require gem_name`, no `require:`
+# option needed in the Gemfile) actually finds it. The real code lives at
+# the conventional-looking paths below (mirroring how Rails organizes its
+# own built-in adapters) -- this file is just Bundler's entrypoint into
+# them. Rails' own ActiveRecord::ConnectionAdapters.register/
+# DatabaseTasks.register_task (bottom of each file below) treat the require
+# path passed to them as an opaque string, not a real convention, so
+# pointing them at this file rather than either individual one is what
+# makes their lazy-load fallback path (loading before Bundler's own
+# implicit require has run) reach both classes, not just one.
+require_relative "active_record/connection_adapters/virgodb_adapter"
+require_relative "active_record/tasks/virgodb_database_tasks"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-virgodb-adapter
 version: !ruby/object:Gem::Version
-  version: 0.1.0.dev.1424cbd
+  version: 0.1.2
 platform: ruby
 authors:
 - virgodb
@@ -23,6 +23,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -39,7 +53,7 @@ dependencies:
         version: '1.4'
 description: |
   Lets any Rails app write real ActiveRecord::Migration subclasses against a virgodb manifest,
-  picked up by `rails db:migrate` the same way SQLite and ClickHouse migrations already are.
+  picked up by `rails db:migrate` the same way SQLite migrations already are.
   Subclasses ActiveRecord::ConnectionAdapters::SQLite3Adapter (virgodb's manifest is already a
   real SQLite file), so structure_dump/the :sql schema format work for free.
 executables: []
@@ -48,6 +62,8 @@ extra_rdoc_files: []
 files:
 - LICENSE
 - lib/active_record/connection_adapters/virgodb_adapter.rb
+- lib/active_record/tasks/virgodb_database_tasks.rb
+- lib/activerecord-virgodb-adapter.rb
 licenses:
 - Nonstandard
 metadata: {}