activerecord-virgodb-adapter 0.1.0.dev.2fa4849

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f4d9953d82781f0f15cc80deb100e9a534ebcabafe894a6cff258767e61c2e0d
+  data.tar.gz: d046ad46492f2de29816497f566095f77d48defe0343f1bf6c44c9a455edc0bf
+SHA512:
+  metadata.gz: afdf198e73fe36aa1cca5036b8b1713dac85a1f62885b9bd079472916d38dc0b6225cfaa9f2564dfe1f806a5489102390284986f6388a7ad765193e59ddac7ba
+  data.tar.gz: fd8ff15112deb4bae56553011e7f2ac6e5d73a3d8847b276fdda719e4480a1f5a8d4b691c0316ee1056aa6d292ca3f9fa316f12fb046ec2423ae7927a8946d7a

data/LICENSE

@@ -0,0 +1,5 @@
+Copyright (c) 2026 Sumit Kumar. All rights reserved.
+
+This software and associated documentation files (the "Software") are proprietary. You may install and use the compiled binary for personal, non-commercial purposes only. Any commercial use, redistribution, modification, or reverse engineering of the Software is strictly prohibited without explicit written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/active_record/connection_adapters/virgodb_adapter.rb

@@ -0,0 +1,383 @@
+# frozen_string_literal: true
+
+require "active_record/connection_adapters/sqlite3_adapter"
+require "json"
+
+module ActiveRecord
+  module ConnectionAdapters
+    # ActiveRecord adapter for virgodb migrations -- lets any Rails app
+    # define and evolve a virgodb table's schema the same way it already
+    # does for SQLite (bin/rails db:migrate) and ClickHouse (the
+    # clickhouse-activerecord gem): real ActiveRecord::Migration subclasses,
+    # raw type strings, picked up by `rails db:migrate` alongside every
+    # other database.
+    #
+    # virgodb's manifest is already a real SQLite file, so subclassing
+    # SQLite3Adapter (not building from AbstractAdapter) gets
+    # structure_dump/structure_load -- the :sql schema format -- for free,
+    # zero custom dumper code. Only create_table/add_column need
+    # overriding: call `super` for a real, if never-written-to, SQLite
+    # table (SQLite's type-name grammar accepts arbitrary bare-word type
+    # declarations -- verified against a real connection, see
+    # crates/compact/src/schema.rs's doc comment for the exact grammar and
+    # why it has to be `"AnyLast String updated_at"`, not
+    # `"AnyLast(String, updated_at)"`), and additionally record the column
+    # in a `schema_versions` table matching exactly what
+    # `compact::schema_from_columns` on the Rust side expects to parse.
+    #
+    # One manifest can back several virgodb tables sharing a single
+    # `database.yml` connection and migrations path -- the same way a host
+    # app's `clickhouse:` connection might already hold all of its ClickHouse
+    # tables under one connection. `schema_versions` (and the Rust-side
+    # manifest's `run_files`/`inline_batches`/`orphaned_files`/
+    # `maintenance_lease` catalog tables) are scoped by `table_name`
+    # specifically to make this safe: each logical table's schema history
+    # and physical run files stay independent even though they live in the
+    # same SQLite file.
+    #
+    # Additive-only, matching `Manifest::register_schema_version`'s own
+    # rule and virgodb's immutable-Parquet philosophy: `remove_column`,
+    # `rename_column`, and `change_column` are all disabled outright rather
+    # than silently letting the real SQLite table drift out of sync with
+    # what `schema_versions` describes.
+    class VirgodbAdapter < SQLite3Adapter
+      ADAPTER_NAME = "Virgodb"
+
+      # Real bug, found (not assumed) by testing: setting `auto_vacuum =
+      # INCREMENTAL` from `create_table` -- even from create_table's very
+      # FIRST call on a brand-new manifest, before any table exists -- was
+      # already too late. SQLite3Adapter's own `configure_connection` sets
+      # `journal_mode = wal` via DEFAULT_PRAGMAS (`super`, below) as part of
+      # establishing the connection itself, which happens before this
+      # adapter's `create_table` ever runs -- and switching journal_mode to
+      # WAL requires SQLite to actually write to the file (creating the -wal
+      # file, an implicit checkpoint), which "poisons" the database from
+      # auto_vacuum's perspective the same way any other content would.
+      # Confirmed directly: a manual `PRAGMA auto_vacuum = INCREMENTAL`
+      # issued immediately after `establish_connection`, before ANY table or
+      # migration ran, still silently failed. The pragma has to be set
+      # before `super` runs here, not merely before the first CREATE TABLE.
+      def configure_connection
+        @raw_connection.auto_vacuum = "incremental" if @raw_connection.respond_to?(:auto_vacuum=)
+        super
+      end
+
+      def create_table(table_name, **options, &block)
+        result = super
+        record_schema_version!(table_name)
+        result
+      end
+
+      def add_column(table_name, column_name, type, **options)
+        result = super
+        record_added_column!(table_name, column_name, type, options)
+        result
+      end
+
+      def remove_column(table_name, column_name, type = nil, **options)
+        raise NotImplementedError, "virgodb schemas are additive-only -- columns can be added, never removed (see schema_versions)"
+      end
+
+      def rename_column(table_name, column_name, new_column_name)
+        raise NotImplementedError, "virgodb schemas are additive-only -- columns can't be renamed once created"
+      end
+
+      def change_column(table_name, column_name, type, **options)
+        raise NotImplementedError, "virgodb schemas are additive-only -- an existing column's type can't change"
+      end
+
+      private
+
+      # Reads the table back via ActiveRecord's own column introspection --
+      # `sql_type` round-trips the raw virgodb type string verbatim, proven
+      # against a real SQLite connection immediately after a fresh CREATE
+      # TABLE -- and writes a new `schema_versions` row: the full current
+      # column list, versioned one higher than whatever was there before FOR
+      # THIS table_name. Mirrors Manifest::register_schema_version's shape
+      # exactly (same table_name scoping, same column names, same JSON shape
+      # for `columns`) so the Rust side can read whatever Ruby wrote with
+      # zero translation. Scoping `MAX(version)` by `table_name` is what makes
+      # it safe for several logical tables to share one manifest/connection:
+      # without it, the second table's first migration would start at
+      # version 2 (or worse, silently share version numbers with an
+      # unrelated table). Rails' OWN bookkeeping tables (`schema_migrations`,
+      # `ar_internal_metadata`) go through this SAME overridden `create_table`
+      # the first time either is lazily created -- there's nothing
+      # virgodb-specific about them, and recording a schema_versions row for
+      # them is pure noise (found while dumping a real structure.sql and
+      # seeing them show up as rows next to the real logical tables). Guarded
+      # here, not by checking `table_name` against `TABLES` in some caller --
+      # this adapter has no such registry, `create_table`'s own argument is
+      # the one place that reliably sees every table, virgodb or not.
+      #
+      # ONLY safe for `create_table` (a brand-new table with no prior
+      # history). NOT reused by `add_column` -- see `record_added_column!`'s
+      # own doc comment for the real bug this avoids: `columns(table_name)`
+      # stops reliably returning the ORIGINAL declared type text for
+      # pre-existing columns once the table has been through certain ALTER
+      # operations.
+      RAILS_INTERNAL_TABLES = %w[schema_migrations ar_internal_metadata].freeze
+
+      def record_schema_version!(table_name)
+        return if RAILS_INTERNAL_TABLES.include?(table_name.to_s)
+
+        ensure_schema_versions_table!
+
+        cols = columns(table_name).map do |c|
+          { "name" => c.name, "virgodb_type" => c.sql_type, "nullable" => c.null }
+        end
+
+        insert_schema_version!(table_name, cols)
+      end
+
+      # `add_column`'s schema_versions update -- deliberately does NOT
+      # re-introspect the whole table via `columns(table_name)` the way
+      # `record_schema_version!` does. Real bug, found (not assumed) by
+      # testing: `super`'s real `ALTER TABLE ADD COLUMN` -- specifically for
+      # a `null: false` column, which SQLite's native ALTER can't add without
+      # a default -- makes Rails' SQLite3Adapter fall back to its "rebuild
+      # the table" strategy (copy to a temp table, drop, rename). Confirmed
+      # directly: introspecting the table immediately afterward, even via a
+      # completely fresh connection/process (so not a Rails-process-level
+      # schema cache artifact), returns Rails' own GENERIC type names for
+      # every PRE-EXISTING column ("INTEGER"/"datetime"/"" instead of
+      # "Int64"/"Timestamp"/"LowCardinality String") -- the rebuild
+      # reconstructs the physical table's DDL from Rails' normalized column
+      # metadata, silently discarding the original literal type text for
+      # every column it didn't just add. Re-deriving the full column list
+      # via introspection after ANY add_column would therefore corrupt every
+      # already-correct entry in `schema_versions`.
+      #
+      # The fix: never re-derive what's already known to be correct. Read
+      # the CURRENT (highest-version) column list straight out of
+      # `schema_versions` itself (the one thing guaranteed not to have been
+      # touched by the ALTER), and append only the ONE new column -- whose
+      # name/type/nullability are already fully known from `add_column`'s own
+      # arguments, no introspection needed for it either.
+      def record_added_column!(table_name, column_name, type, options)
+        return if RAILS_INTERNAL_TABLES.include?(table_name.to_s)
+
+        ensure_schema_versions_table!
+
+        current_json = select_value("SELECT columns FROM schema_versions WHERE table_name = #{quote(table_name)} ORDER BY version DESC LIMIT 1")
+        current_cols = current_json ? JSON.parse(current_json) : []
+
+        new_col = { "name" => column_name.to_s, "virgodb_type" => type.to_s, "nullable" => options[:null] != false }
+        insert_schema_version!(table_name, current_cols + [ new_col ])
+      end
+
+      def insert_schema_version!(table_name, cols)
+        next_version = (select_value("SELECT MAX(version) FROM schema_versions WHERE table_name = #{quote(table_name)}") || 0).to_i + 1
+
+        execute(<<~SQL)
+          INSERT INTO schema_versions (table_name, version, columns, applied_at)
+          VALUES (#{quote(table_name)}, #{next_version}, #{quote(cols.to_json)}, #{Time.now.to_i})
+        SQL
+      end
+
+      # Same DDL as Manifest::open()'s bootstrap on the Rust side (same
+      # table/column names, including the `table_name` scoping column) --
+      # IF NOT EXISTS makes this safe to call whether virgodb's own Rust
+      # code or this Ruby adapter created the manifest file first. Plain
+      # INTEGER PRIMARY KEY, not AUTOINCREMENT -- see the matching comment
+      # in manifest/src/lib.rs for why (breaks a real structure_dump/
+      # structure_load round trip the moment any host app's
+      # ActiveRecord::SchemaDumper.ignore_tables is non-empty).
+      def ensure_schema_versions_table!
+        execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS schema_versions (
+            id         INTEGER PRIMARY KEY,
+            table_name TEXT NOT NULL,
+            version    INTEGER NOT NULL,
+            columns    TEXT NOT NULL,
+            applied_at INTEGER NOT NULL
+          )
+        SQL
+      end
+    end
+  end
+end
+
+ActiveRecord::ConnectionAdapters.register(
+  "virgodb",
+  "ActiveRecord::ConnectionAdapters::VirgodbAdapter",
+  "active_record/connection_adapters/virgodb_adapter"
+)
+
+module ActiveRecord
+  module Tasks
+    # `ActiveRecord::Tasks::SQLiteDatabaseTasks#structure_dump` just shells
+    # out to the real `sqlite3` CLI (`.schema --nosys`, or a raw `SELECT sql
+    # FROM sqlite_master` when a host app has any
+    # `ActiveRecord::SchemaDumper.ignore_tables` configured -- e.g. excluding
+    # tables owned by a separate replication/backup mechanism) against the
+    # configured database file --
+    # DDL only. `ActiveRecord::Tasks::DatabaseTasks.dump_schema` then
+    # special-cases exactly one table on top of that: if `schema_migrations`
+    # exists, it appends that table's real row data (`connection
+    # .dump_schema_versions`) to the same dump file, because Rails' own
+    # migration bookkeeping needs those rows to survive a `db:schema:load`/
+    # `structure_load`, not just the table shape.
+    #
+    # `schema_versions` (this adapter's own bookkeeping table, plural,
+    # distinct from Rails' singular `schema_migrations`) needs exactly the
+    # same treatment and doesn't get it for free: it holds real data (the
+    # versioned column list `compact::schema_from_columns` on the Rust side
+    # depends on), not just structure, and Rails has no way to know that a
+    # table it didn't create needs the same row-preserving special case.
+    # Confirmed empirically: without this, a fresh database prepared via
+    # `structure_load` (which Rails does automatically whenever a `db:migrate`
+    # invocation for a later environment/connection finds an existing,
+    # version-matching dump already on disk from an earlier one -- not
+    # something this adapter can opt out of) ends up with the real
+    # host-app table but an EMPTY `schema_versions`, so
+    # `Manifest::current_schema_version()` on the Rust side returns `None`
+    # even though the SQLite table itself is correct.
+    class VirgodbDatabaseTasks < SQLiteDatabaseTasks
+      class << self
+        # Optional host-app hook: `->(sql_table_name) { {sort_by:, key_column:,
+        # partition_column:} or nil }`. Purely a documentation comment written
+        # above CREATE TABLE at dump time -- nothing on the Rust or Ruby side
+        # ever reads it back. This adapter gem stays host-app-agnostic (see
+        # the class doc comment above: "lets any Rails app..."), so it can't
+        # hardcode any one host app's sort_by/key_column config (that's
+        # app/services/virgodb/registry.rb's job, which already has this
+        # exact data for real -- passed to Virgodb.start_maintenance on every
+        # boot). A host app that wants it surfaced sets this once, e.g. in an
+        # initializer; left nil, structure_dump behaves exactly as before.
+        attr_accessor :table_layout_provider
+      end
+
+      def structure_dump(filename, extra_flags)
+        super
+        reformat_create_table_statements!(filename)
+        annotate_physical_layout!(filename)
+        dump_schema_versions!(filename)
+      end
+
+      private
+
+      # Only the LATEST schema_versions row per table_name is dumped.
+      # `Manifest::current_schema_version` (manifest/src/lib.rs) only ever
+      # reads `ORDER BY version DESC LIMIT 1` -- confirmed directly, no Rust
+      # or Ruby code anywhere reads an older version row, so keeping full
+      # history here isn't a correctness requirement. It also isn't a
+      # readability win: each version's `columns` re-lists every column from
+      # scratch, so N migrations means N near-duplicate JSON blobs, longer
+      # every time. The version count/dates are still worth one line for
+      # orientation -- the full column list per version isn't; the
+      # migration files themselves (db/migrate_virgodb_*/*.rb, timestamped,
+      # under git blame) are already the authoritative "what changed when"
+      # record. `register_schema_version`'s additive-only check is
+      # unaffected: it only ever compares a new migration's proposed columns
+      # against the CURRENT version, which is exactly what's still dumped.
+      def dump_schema_versions!(filename)
+        return unless connection.data_source_exists?("schema_versions")
+
+        rows = connection.select_all("SELECT id, table_name, version, columns, applied_at FROM schema_versions ORDER BY table_name, version")
+        return if rows.to_a.empty?
+
+        File.open(filename, "a") do |f|
+          f.puts
+          rows.to_a.group_by { |r| r["table_name"] }.each do |table_name, versions|
+            latest = versions.max_by { |r| r["version"] }
+            history = versions.map { |r| "v#{r['version']} (#{Time.at(r['applied_at'].to_i).utc.strftime('%Y-%m-%d')})" }.join(", ")
+
+            f.puts "-- #{table_name} schema history: #{history} -- only the current version is dumped below; see db/migrate_virgodb_*/ for the rest"
+            pretty_columns = JSON.pretty_generate(JSON.parse(latest["columns"]))
+            f.puts(
+              "INSERT INTO \"schema_versions\" (id, table_name, version, columns, applied_at) VALUES (\n" \
+              "  #{latest['id']}, #{connection.quote(table_name)}, #{latest['version']},\n" \
+              "  #{connection.quote(pretty_columns)},\n" \
+              "  #{latest['applied_at']}\n" \
+              ");"
+            )
+            f.puts
+          end
+        end
+      end
+
+      # Writes `table_layout_provider`'s sort_by/key_column/partition_column
+      # as a comment directly above each table's (already reformatted,
+      # one-column-per-line) CREATE TABLE -- the closest a SQLite-flavored
+      # dump can get to ClickHouse's own `ORDER BY`/`PARTITION BY` clauses
+      # being right there in the DDL. No-op (and no dump format change at
+      # all) when no provider is registered.
+      def annotate_physical_layout!(filename)
+        provider = self.class.table_layout_provider
+        return unless provider
+
+        content = File.read(filename)
+        content.gsub!(/^CREATE TABLE (IF NOT EXISTS )?"?(\w+)"? \(/) do
+          statement_start = Regexp.last_match(0)
+          table_name = Regexp.last_match(2)
+          layout = provider.call(table_name)
+          next statement_start unless layout
+
+          comment = [
+            "-- Physical layout (app/services/virgodb/registry.rb):",
+            "--   sort_by:          #{layout[:sort_by].join(', ')}",
+            "--   key_column:       #{layout[:key_column]} (run file min/max pruning bounds)",
+            "--   partition_column: #{layout[:partition_column]}"
+          ].join("\n")
+
+          "#{comment}\n#{statement_start}"
+        end
+        File.write(filename, content)
+      end
+
+      # `SQLite3Adapter#structure_dump`'s raw `SELECT sql FROM sqlite_master`
+      # path (used whenever `ActiveRecord::SchemaDumper.ignore_tables` is
+      # non-empty, e.g. excluding tables owned by a separate replication/
+      # backup mechanism) reproduces each
+      # CREATE TABLE exactly as ActiveRecord originally generated it --
+      # one long line, every column crammed together. Fine for SQLite to
+      # load back (structure_load doesn't care about whitespace), useless to
+      # a human reading the dump next to `db/clickhouse_structure.sql`
+      # (ClickHouse's own `SHOW CREATE TABLE` is naturally one-column-per-line,
+      # for comparison). Purely cosmetic, safe to do unconditionally: only
+      # rewrites the exact same statement with different whitespace, no DDL
+      # semantics change. Skips statements that already span multiple lines
+      # (e.g. this adapter's own hand-written `schema_versions` DDL, created
+      # via a real newline-containing heredoc) -- nothing to reformat there.
+      def reformat_create_table_statements!(filename)
+        content = File.read(filename)
+        content.gsub!(/^CREATE TABLE (IF NOT EXISTS )?("[\w]+"|\w+) \((.*)\);$/) do
+          if_not_exists = Regexp.last_match(1)
+          table_name = Regexp.last_match(2)
+          columns = split_top_level_commas(Regexp.last_match(3))
+          next Regexp.last_match(0) if columns.size <= 1
+
+          "CREATE TABLE #{if_not_exists}#{table_name} (\n" + columns.map { |c| "  #{c.strip}" }.join(",\n") + "\n);"
+        end
+        File.write(filename, content)
+      end
+
+      # Splits a CREATE TABLE column list on commas that are NOT nested
+      # inside a type's own parens (e.g. `Decimal(18,2)`) -- a plain
+      # `String#split(",")` would wrongly cut `Decimal(18` and `2)` apart.
+      def split_top_level_commas(str)
+        parts = []
+        depth = 0
+        current = +""
+        str.each_char do |ch|
+          case ch
+          when "(" then depth += 1; current << ch
+          when ")" then depth -= 1; current << ch
+          when ","
+            depth.zero? ? (parts << current; current = +"") : current << ch
+          else
+            current << ch
+          end
+        end
+        parts << current unless current.strip.empty?
+        parts
+      end
+    end
+  end
+end
+
+# Registering our own subclass (not plain SQLiteDatabaseTasks) so
+# schema_versions' row data survives the dump/load round trip -- see the
+# class doc comment above for why a real Rails app can't avoid hitting this.
+ActiveRecord::Tasks::DatabaseTasks.register_task(/virgodb/, "ActiveRecord::Tasks::VirgodbDatabaseTasks")

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification
+name: activerecord-virgodb-adapter
+version: !ruby/object:Gem::Version
+  version: 0.1.0.dev.2fa4849
+platform: ruby
+authors:
+- virgodb
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.4'
+description: |
+  Lets any Rails app write real ActiveRecord::Migration subclasses against a virgodb manifest,
+  picked up by `rails db:migrate` the same way SQLite and ClickHouse migrations already are.
+  Subclasses ActiveRecord::ConnectionAdapters::SQLite3Adapter (virgodb's manifest is already a
+  real SQLite file), so structure_dump/the :sql schema format work for free.
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- lib/active_record/connection_adapters/virgodb_adapter.rb
+licenses:
+- Nonstandard
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.16
+specification_version: 4
+summary: ActiveRecord adapter for defining and evolving virgodb table schemas via
+  real migrations
+test_files: []