activerecord-trino-adapter 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b38f0bee0dc9764f269632d9b138de7aff8c4fb3fb696c8862bd8de23a34999e
-  data.tar.gz: 8a24ba5209ee11e82ee43d67182798599b96b1a9f1a621b4f60edf0a2ae3c0ed
+  metadata.gz: 1371d47ebd2cce2d35ff2390bcc4fb825a851c222cb6a445ece9dcfcd5eb47d7
+  data.tar.gz: 705759f9bc0613d910025bd238b3c8618c05aa6a51525085aeeb9301605a438a
 SHA512:
-  metadata.gz: 83cba7a9d4c291d021d94729bd2e1fbb37e101ec19749163b4eebcee99969360cd46e3831e0d952eb63770d6db53e46abce9627fa82208f3103b38361c15148b
-  data.tar.gz: fd39bfe9b1025178a0a95e35b03bbd9617a3859931e7eea02af7a8ce96a8cf02dd7090469fe08ceca19208a1c9f53a20d83345ca78229e2adb426408102fa624
+  metadata.gz: 89048af018a7bc867bef4f34ec3eb585d021fb85019e4c1b125746c9b751949dbbbcdb975b9077888b69cb423e561a43e68a65a858d363b99f4398d528dda46b
+  data.tar.gz: d5db090fb26309668cf9859ff46224f91fef8a05318fef993bf1f20866b9e56b5faac98382bf99e77cffe28919b140f9c176013360f0718b622e20dfe092d689

data/README.md

@@ -87,6 +87,65 @@ All keys are read from the `database.yml` entry:
 | `query_timeout` | `150` | Hard ceiling on query duration, in seconds. Cap lower for user-facing paths and higher for backfills |
 | `plan_timeout` | `30` | Ceiling on Trino query-planning phase, in seconds |
 | `slow_query_threshold_seconds` | `20` | Threshold above which an `active_record_trino.slow_query` notification is emitted |
+| `persistent` | `false` | Reuse one keep-alive HTTP connection per adapter instance instead of opening a fresh TCP+TLS connection for every request. See [Persistent HTTP connections](#persistent-http-connections) |
+| `gzip` | _nil_ | When `true`, requests gzip-compressed HTTP response bodies from Trino |
+| `bulk_column_reflection` | `false` | Reflect every table's columns in a single `information_schema.columns` query instead of one per table. See [Bulk column reflection](#bulk-column-reflection) |
+| `static_schema` | `false` | Serve table existence from columns declared via `ActiveRecord::Trino.define_columns` instead of running `SHOW TABLES`. See [Static schema declarations](#static-schema-declarations) |
+
+### Persistent HTTP connections
+
+A single Trino query is 4-6 HTTP requests (`POST /v1/statement`, then repeated
+`nextUri` polls), and by default each one pays a full TCP + TLS handshake. With
+`persistent: true` the adapter keeps one keep-alive connection per adapter
+instance (Rails checks out one adapter instance per thread, so no locking is
+involved) and reuses it across requests and queries. Against a TLS-fronted
+cluster this typically saves several hundred milliseconds per query.
+
+Idle keep-alive sockets are recycled after 100 seconds, below typical
+load-balancer idle timeouts, and Trino protocol GET polls are retried
+transparently if the server closes a kept-alive socket. `disconnect!` shuts the
+pool down; `reconnect!` rebuilds it.
+
+### Bulk column reflection
+
+By default the adapter reflects a model's columns with a single-table
+`information_schema.columns` query the first time ActiveRecord loads its schema.
+With `bulk_column_reflection: true`, the first reflection instead issues one
+`information_schema.columns` query for the whole catalog/schema and groups the
+result by table, memoized per connection. Since each Trino query carries a few
+hundred milliseconds of fixed overhead, reflecting N tables this way costs one
+round trip instead of N — useful when a process touches many warehouse tables.
+The cache is cleared on `reconnect!` and by `ActiveRecord::Trino.reset_schema_cache!`.
+
+### Static schema declarations
+
+If your team owns the warehouse schema, you can declare a table's columns in
+code and skip reflection entirely. Register columns with
+`ActiveRecord::Trino.define_columns`:
+
+```ruby
+ActiveRecord::Trino.define_columns("sales_by_day", [
+  { name: :territory_id, sql_type: "integer", null: false },
+  { name: :month,        sql_type: "date" },
+  { name: :amount,       sql_type: "decimal(18, 2)" }, # null defaults to true
+])
+```
+
+When a table is registered, `#columns` serves the declared definitions and never
+queries `information_schema`. Use the Trino SQL type strings (`bigint`,
+`integer`, `date`, `decimal(18, 2)`, `varchar`, `timestamp(3)`, `boolean`, …) —
+the same values `information_schema` would return — so type casting is identical.
+Declaration is per table: undeclared tables still reflect.
+
+Setting `static_schema: true` additionally serves table existence
+(`#data_sources`) from the registered tables, so `SHOW TABLES` is never run. With
+this enabled, every table the app queries must be declared, or ActiveRecord will
+treat it as nonexistent.
+
+Declarations become the source of truth: a column added, dropped, or retyped in
+the warehouse is not picked up until you update the declaration and redeploy
+(unlike reflection, which self-corrects on reconnect). Best suited to schemas
+your team controls and changes deliberately.
 
 ## Instrumentation
 

data/lib/active_record/connection_adapters/trino/database_statements.rb

@@ -53,7 +53,7 @@ module ActiveRecord
         # rubocop:disable Metrics/AbcSize
         def run_trino_query(sql)
           start = monotonic_now
-          query = client.query(sql)
+          query = start_query(sql)
           internal = consume_query(query)
           capture_query_metadata(query)
           notify_slow_query(sql, monotonic_now - start)
@@ -69,6 +69,13 @@ module ActiveRecord
         end
         # rubocop:enable Metrics/AbcSize
 
+        def start_query(sql)
+          return client.query(sql) unless persistent?
+
+          statement = ::Trino::Client::StatementClient.new(persistent_faraday, sql, @client_options)
+          ::Trino::Client::Query.new(statement)
+        end
+
         def consume_query(query)
           columns = query.columns || []
           rows = []

data/lib/active_record/connection_adapters/trino/schema_statements.rb

@@ -5,18 +5,20 @@ module ActiveRecord
     module Trino
       module SchemaStatements
         def columns(table_name)
-          rows = run_trino_query(columns_query(table_name.to_s)).rows
-          rows.map do |name, data_type, is_nullable|
-            Trino::Column.new(
-              name: name,
-              sql_type: data_type,
-              type: type_map.lookup(data_type),
-              null: nullable?(is_nullable)
-            )
+          table = table_name.to_s
+          declared = ActiveRecord::Trino.static_columns[table]
+          return build_static_columns(declared) if declared
+
+          if bulk_column_reflection?
+            column_definitions.fetch(table, [])
+          else
+            build_columns(run_trino_query(table_columns_query(table)).rows)
           end
         end
 
         def data_sources
+          return ActiveRecord::Trino.static_columns.keys if static_schema?
+
           run_trino_query("SHOW TABLES").rows.map(&:first)
         end
         alias tables data_sources
@@ -46,13 +48,46 @@ module ActiveRecord
           false
         end
 
-        def schema_cache
-          @schema_cache ||= ActiveRecord::ConnectionAdapters::SchemaCache.new(self)
+        def clear_column_cache!
+          @column_definitions = nil
         end
 
       private
 
-        def columns_query(table_name)
+        def build_columns(rows)
+          rows.map do |name, data_type, is_nullable|
+            Trino::Column.new(
+              name: name,
+              sql_type: data_type,
+              type: type_map.lookup(data_type),
+              null: nullable?(is_nullable)
+            )
+          end
+        end
+
+        def build_static_columns(definitions)
+          definitions.map do |definition|
+            sql_type = definition.fetch(:sql_type)
+            Trino::Column.new(
+              name: definition.fetch(:name).to_s,
+              sql_type: sql_type,
+              type: type_map.lookup(sql_type),
+              null: definition.fetch(:null, true)
+            )
+          end
+        end
+
+        def column_definitions
+          @column_definitions ||= load_column_definitions
+        end
+
+        def load_column_definitions
+          run_trino_query(all_columns_query).rows.group_by(&:first).transform_values do |rows|
+            build_columns(rows.map { |row| row.drop(1) })
+          end
+        end
+
+        def table_columns_query(table_name)
           <<~SQL.strip
             SELECT column_name, data_type, is_nullable
             FROM information_schema.columns
@@ -63,6 +98,16 @@ module ActiveRecord
           SQL
         end
 
+        def all_columns_query
+          <<~SQL.strip
+            SELECT table_name, column_name, data_type, is_nullable
+            FROM information_schema.columns
+            WHERE table_catalog = #{quote(trino_catalog)}
+              AND table_schema = #{quote(trino_schema)}
+            ORDER BY table_name, ordinal_position
+          SQL
+        end
+
         def trino_catalog
           @client_options[:catalog]
         end

data/lib/active_record/connection_adapters/trino_adapter.rb

@@ -2,6 +2,7 @@
 
 require "active_record"
 require "active_record/connection_adapters/abstract_adapter"
+require "faraday/net_http_persistent"
 require "trino-client"
 
 require "active_record/trino"
@@ -19,6 +20,8 @@ module ActiveRecord
     class TrinoAdapter < AbstractAdapter
       ADAPTER_NAME = "Trino"
 
+      PERSISTENT_IDLE_TIMEOUT = 100
+
       include Trino::Quoting
       include Trino::DatabaseStatements
       include Trino::SchemaStatements
@@ -28,6 +31,9 @@ module ActiveRecord
         super
         @client_options = ActiveRecord::Trino::Config.client_options(@config)
         @slow_query_threshold = ActiveRecord::Trino::Config.slow_query_threshold(@config)
+        @persistent = ActiveRecord::Trino::Config.persistent?(@config)
+        @bulk_column_reflection = ActiveRecord::Trino::Config.bulk_column_reflection?(@config)
+        @static_schema = ActiveRecord::Trino::Config.static_schema?(@config)
         @client = build_client
         install_safety_belts!
       end
@@ -46,7 +52,10 @@ module ActiveRecord
       end
 
       def disconnect!
+        @persistent_faraday&.close
+        @persistent_faraday = nil
         @client = nil
+        clear_column_cache!
       end
 
       def supports_transactions?
@@ -104,12 +113,36 @@ module ActiveRecord
 
       attr_reader :client, :last_query_id, :last_query_info_uri, :last_query_stats
 
+      def persistent?
+        @persistent
+      end
+
+      def bulk_column_reflection?
+        @bulk_column_reflection
+      end
+
+      def static_schema?
+        @static_schema
+      end
+
     private
 
       def build_client
         ::Trino::Client.new(@client_options)
       end
 
+      def persistent_faraday
+        @persistent_faraday ||= build_persistent_faraday
+      end
+
+      def build_persistent_faraday
+        faraday = ::Trino::Client.faraday_client(@client_options)
+        faraday.builder.adapter(:net_http_persistent) do |http|
+          http.idle_timeout = PERSISTENT_IDLE_TIMEOUT
+        end
+        faraday
+      end
+
       def type_map
         @type_map ||= Trino::TypeMap.build
       end

data/lib/active_record/trino/config.rb

@@ -27,11 +27,24 @@ module ActiveRecord
           ssl: ssl,
           http_proxy: symbolized[:http_proxy],
           time_zone: symbolized[:time_zone],
+          gzip: symbolized[:gzip],
           query_timeout: symbolized.fetch(:query_timeout, DEFAULT_QUERY_TIMEOUT),
           plan_timeout: symbolized.fetch(:plan_timeout, DEFAULT_PLAN_TIMEOUT),
         }.compact
       end
 
+      def persistent?(config)
+        !!symbolize(config).fetch(:persistent, false)
+      end
+
+      def bulk_column_reflection?(config)
+        !!symbolize(config).fetch(:bulk_column_reflection, false)
+      end
+
+      def static_schema?(config)
+        !!symbolize(config).fetch(:static_schema, false)
+      end
+
       def default_port(ssl)
         ssl ? DEFAULT_HTTPS_PORT : DEFAULT_HTTP_PORT
       end

data/lib/active_record/trino/version.rb

@@ -2,6 +2,6 @@
 
 module ActiveRecord
   module Trino
-    VERSION = "0.1.0"
+    VERSION = "0.2.0"
   end
 end

data/lib/active_record/trino.rb

@@ -31,8 +31,21 @@ end
 
 module ActiveRecord
   module Trino
+    def self.static_columns
+      @static_columns ||= {}
+    end
+
+    def self.define_columns(table_name, definitions)
+      static_columns[table_name.to_s] = Array(definitions)
+    end
+
     def self.reset_schema_cache!(model_class)
       model_class.reset_column_information
+
+      model_class.connection_pool.connections.each do |conn|
+        conn.clear_column_cache! if conn.respond_to?(:clear_column_cache!)
+      end
+
       return unless model_class.connection.respond_to?(:schema_cache)
 
       model_class.connection.schema_cache.clear!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-trino-adapter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Garett Arrowood
@@ -49,6 +49,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '8.1'
+- !ruby/object:Gem::Dependency
+  name: faraday-net_http_persistent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: trino-client
   requirement: !ruby/object:Gem::Requirement