activerecord-materialize-adapter 0.2.0 → 0.3.0

data/lib/active_record/connection_adapters/materialize/version.rb

@@ -3,7 +3,7 @@
 module ActiveRecord
   module ConnectionAdapters
     module Materialize
-      VERSION = "0.2.0"
+      VERSION = "0.3.0"
     end
   end
 end

data/lib/active_record/connection_adapters/materialize_adapter.rb

@@ -29,9 +29,12 @@ require "active_record/connection_adapters/materialize/schema_creation"
 require "active_record/connection_adapters/materialize/schema_definitions"
 require "active_record/connection_adapters/materialize/schema_dumper"
 require "active_record/connection_adapters/materialize/schema_statements"
+require "active_record/connection_adapters/materialize/schema/source_statements"
+require "active_record/connection_adapters/materialize/schema/view_statements"
 require "active_record/connection_adapters/materialize/type_metadata"
 require "active_record/connection_adapters/materialize/utils"
 require "active_record/tasks/materialize_database_tasks"
+require 'materialize/errors/incomplete_input'
 
 module ActiveRecord
   module ConnectionHandling # :nodoc:
@@ -105,45 +108,24 @@ module ActiveRecord
       class_attribute :create_unlogged_tables, default: false
 
       NATIVE_DATABASE_TYPES = {
-        primary_key: "bigserial primary key",
+        primary_key: "bigint",
         string:      { name: "character varying" },
         text:        { name: "text" },
-        integer:     { name: "integer", limit: 4 },
-        float:       { name: "float" },
+        integer:     { name: "integer" },
+        bigint:      { name: "int8" },
+        float:       { name: "float4" },
+        double:      { name: "float8" },
         decimal:     { name: "decimal" },
         datetime:    { name: "timestamp" },
+        timestampz:  { name: "timestamp with time zone" },
         time:        { name: "time" },
         date:        { name: "date" },
-        daterange:   { name: "daterange" },
-        numrange:    { name: "numrange" },
-        tsrange:     { name: "tsrange" },
-        tstzrange:   { name: "tstzrange" },
-        int4range:   { name: "int4range" },
-        int8range:   { name: "int8range" },
         binary:      { name: "bytea" },
         boolean:     { name: "boolean" },
-        xml:         { name: "xml" },
-        tsvector:    { name: "tsvector" },
-        hstore:      { name: "hstore" },
-        inet:        { name: "inet" },
-        cidr:        { name: "cidr" },
-        macaddr:     { name: "macaddr" },
-        uuid:        { name: "uuid" },
-        json:        { name: "json" },
-        jsonb:       { name: "jsonb" },
-        ltree:       { name: "ltree" },
-        citext:      { name: "citext" },
-        point:       { name: "point" },
-        line:        { name: "line" },
-        lseg:        { name: "lseg" },
-        box:         { name: "box" },
-        path:        { name: "path" },
-        polygon:     { name: "polygon" },
-        circle:      { name: "circle" },
-        bit:         { name: "bit" },
-        bit_varying: { name: "bit varying" },
-        money:       { name: "money" },
+        map:         { name: "map" },
+        json:        { name: "jsonb" },
         interval:    { name: "interval" },
+        record:      { name: "record" },
         oid:         { name: "oid" },
       }
 
@@ -153,74 +135,31 @@ module ActiveRecord
       include Materialize::ReferentialIntegrity
       include Materialize::SchemaStatements
       include Materialize::DatabaseStatements
+      include Materialize::Schema::SourceStatements
+      include Materialize::Schema::ViewStatements
 
       def supports_bulk_alter?
         true
       end
 
-      def supports_index_sort_order?
-        true
-      end
-
-      def supports_partitioned_indexes?
-        database_version >= 110_000
-      end
-
-      def supports_partial_index?
-        true
-      end
-
-      def supports_expression_index?
-        true
-      end
-
+      # TODO: test transaction isolation
       def supports_transaction_isolation?
         true
       end
 
-      def supports_foreign_keys?
-        true
-      end
-
-      def supports_validate_constraints?
-        true
-      end
-
       def supports_views?
         true
       end
 
-      def supports_datetime_with_precision?
-        true
-      end
-
       def supports_json?
         true
       end
 
-      def supports_comments?
-        true
-      end
-
-      def supports_savepoints?
-        true
-      end
-
+      # TODO: test insert returning
       def supports_insert_returning?
         true
       end
 
-      def supports_insert_on_conflict?
-        database_version >= 90500
-      end
-      alias supports_insert_on_duplicate_skip? supports_insert_on_conflict?
-      alias supports_insert_on_duplicate_update? supports_insert_on_conflict?
-      alias supports_insert_conflict_target? supports_insert_on_conflict?
-
-      def index_algorithms
-        { concurrently: "CONCURRENTLY" }
-      end
-
       class StatementPool < ConnectionAdapters::StatementPool # :nodoc:
         def initialize(connection, max)
           super(max)
@@ -266,7 +205,6 @@ module ActiveRecord
         @type_map = Type::HashLookupTypeMap.new
         initialize_type_map
         @local_tz = execute("SHOW TIMEZONE", "SCHEMA").first["TimeZone"]
-        @use_insert_returning = @config.key?(:insert_returning) ? self.class.type_cast_config_to_boolean(@config[:insert_returning]) : true
       end
 
       def self.database_exists?(config)
@@ -339,10 +277,6 @@ module ActiveRecord
         true
       end
 
-      def supports_extensions?
-        true
-      end
-
       def supports_ranges?
         true
       end
@@ -352,14 +286,6 @@ module ActiveRecord
         true
       end
 
-      def supports_foreign_tables?
-        true
-      end
-
-      def supports_pgcrypto_uuid?
-        database_version >= 90400
-      end
-
       def supports_optimizer_hints?
         unless defined?(@has_pg_hint_plan)
           @has_pg_hint_plan = extension_available?("pg_hint_plan")
@@ -389,33 +315,9 @@ module ActiveRecord
         query_value("SELECT pg_advisory_unlock(#{lock_id})")
       end
 
-      def enable_extension(name)
-        exec_query("CREATE EXTENSION IF NOT EXISTS \"#{name}\"").tap {
-          reload_type_map
-        }
-      end
-
-      def disable_extension(name)
-        exec_query("DROP EXTENSION IF EXISTS \"#{name}\" CASCADE").tap {
-          reload_type_map
-        }
-      end
-
-      def extension_available?(name)
-        query_value("SELECT true FROM pg_available_extensions WHERE name = #{quote(name)}", "SCHEMA")
-      end
-
-      def extension_enabled?(name)
-        query_value("SELECT installed_version IS NOT NULL FROM pg_available_extensions WHERE name = #{quote(name)}", "SCHEMA")
-      end
-
-      def extensions
-        exec_query("SELECT extname FROM pg_extension", "SCHEMA").cast_values
-      end
-
       # Returns the configured supported identifier length supported by Materialize
       def max_identifier_length
-        @max_identifier_length ||= query_value("SHOW max_identifier_length", "SCHEMA").to_i
+        @max_identifier_length || 64
       end
 
       # Set the authorized user for this session
@@ -424,10 +326,6 @@ module ActiveRecord
         execute("SET SESSION AUTHORIZATION #{user}")
       end
 
-      def use_insert_returning?
-        @use_insert_returning
-      end
-
       def column_name_for_operation(operation, node) # :nodoc:
         OPERATION_ALIASES.fetch(operation) { operation.downcase }
       end
@@ -520,44 +418,52 @@ module ActiveRecord
           }
         end
 
+        def load_additional_types(oids = nil)
+          initializer = OID::TypeMapInitializer.new(type_map)
+
+          query = <<~SQL
+            SELECT t.id, t.oid, t.schema_id, t.name
+            FROM mz_types as t
+            LEFT JOIN mz_schemas s ON t.schema_id = s.id
+            LEFT JOIN mz_databases d ON s.database_id = d.id
+            WHERE
+              (d.name = #{quote(current_database)} OR s.name = 'pg_catalog')
+          SQL
+
+          if oids
+            query += "AND t.oid IN (%s)" % oids.join(", ")
+          else
+            query += initializer.query_conditions_for_initial_load
+          end
+
+          execute_and_clear(query, "SCHEMA", []) do |records|
+            initializer.run(records)
+          end
+        end
+
         def initialize_type_map(m = type_map)
           m.register_type "int2", Type::Integer.new(limit: 2)
           m.register_type "int4", Type::Integer.new(limit: 4)
           m.register_type "int8", Type::Integer.new(limit: 8)
+          m.alias_type "int", "int4"
+          m.alias_type "integer", "int4"
+          m.alias_type "bigint", "int8"
           m.register_type "oid", OID::Oid.new
           m.register_type "float4", Type::Float.new
           m.alias_type "float8", "float4"
+          m.alias_type "double", "float4"
           m.register_type "text", Type::Text.new
-          register_class_with_limit m, "varchar", Type::String
-          m.alias_type "char", "varchar"
-          m.alias_type "name", "varchar"
-          m.alias_type "bpchar", "varchar"
-          m.register_type "bool", Type::Boolean.new
-          register_class_with_limit m, "bit", OID::Bit
-          register_class_with_limit m, "varbit", OID::BitVarying
+          m.alias_type "varchar", "text"
+
+          m.register_type "boolean", Type::Boolean.new
+          register_class_with_limit m, "bytea", OID::BitVarying
+
           m.alias_type "timestamptz", "timestamp"
           m.register_type "date", OID::Date.new
 
-          m.register_type "money", OID::Money.new
           m.register_type "bytea", OID::Bytea.new
-          m.register_type "point", OID::Point.new
-          m.register_type "hstore", OID::Hstore.new
-          m.register_type "json", Type::Json.new
           m.register_type "jsonb", OID::Jsonb.new
-          m.register_type "cidr", OID::Cidr.new
-          m.register_type "inet", OID::Inet.new
-          m.register_type "uuid", OID::Uuid.new
-          m.register_type "xml", OID::Xml.new
-          m.register_type "tsvector", OID::SpecializedString.new(:tsvector)
-          m.register_type "macaddr", OID::SpecializedString.new(:macaddr)
-          m.register_type "citext", OID::SpecializedString.new(:citext)
-          m.register_type "ltree", OID::SpecializedString.new(:ltree)
-          m.register_type "line", OID::SpecializedString.new(:line)
-          m.register_type "lseg", OID::SpecializedString.new(:lseg)
-          m.register_type "box", OID::SpecializedString.new(:box)
-          m.register_type "path", OID::SpecializedString.new(:path)
-          m.register_type "polygon", OID::SpecializedString.new(:polygon)
-          m.register_type "circle", OID::SpecializedString.new(:circle)
+          m.alias_type "json", "jsonb"
 
           m.register_type "interval" do |_, _, sql_type|
             precision = extract_precision(sql_type)
@@ -567,25 +473,7 @@ module ActiveRecord
           register_class_with_precision m, "time", Type::Time
           register_class_with_precision m, "timestamp", OID::DateTime
 
-          m.register_type "numeric" do |_, fmod, sql_type|
-            precision = extract_precision(sql_type)
-            scale = extract_scale(sql_type)
-
-            # The type for the numeric depends on the width of the field,
-            # so we'll do something special here.
-            #
-            # When dealing with decimal columns:
-            #
-            # places after decimal  = fmod - 4 & 0xffff
-            # places before decimal = (fmod - 4) >> 16 & 0xffff
-            if fmod && (fmod - 4 & 0xffff).zero?
-              # FIXME: Remove this class, and the second argument to
-              # lookups on PG
-              Type::DecimalWithoutScale.new(precision: precision)
-            else
-              OID::Decimal.new(precision: precision, scale: scale)
-            end
-          end
+          load_additional_types
         end
 
         # Extracts the value from a Materialize column default definition.
@@ -623,26 +511,6 @@ module ActiveRecord
           !default_value && %r{\w+\(.*\)|\(.*\)::\w+|CURRENT_DATE|CURRENT_TIMESTAMP}.match?(default)
         end
 
-        def load_additional_types(oids = nil)
-          initializer = OID::TypeMapInitializer.new(type_map)
-
-          query = <<~SQL
-            SELECT t.oid, t.typname, t.typelem, t.typinput, r.rngsubtype, t.typtype, t.typbasetype
-            FROM pg_type as t
-            LEFT JOIN pg_range as r ON oid = rngtypid
-          SQL
-
-          if oids
-            query += "WHERE t.oid IN (%s)" % oids.join(", ")
-          else
-            query += initializer.query_conditions_for_initial_load
-          end
-
-          execute_and_clear(query, "SCHEMA", []) do |records|
-            initializer.run(records)
-          end
-        end
-
         FEATURE_NOT_SUPPORTED = "0A000" #:nodoc:
 
         def execute_and_clear(sql, name, binds, prepare: false)
@@ -652,10 +520,8 @@ module ActiveRecord
 
           if without_prepared_statement?(binds)
             result = exec_no_cache(sql, name, [])
-          elsif !prepare
-            result = exec_no_cache(sql, name, binds)
           else
-            result = exec_cache(sql, name, binds)
+            result = exec_no_cache(sql, name, binds)
           end
           ret = yield result
           result.clear
@@ -669,39 +535,18 @@ module ActiveRecord
           # made since we established the connection
           update_typemap_for_default_timezone
 
-          type_casted_binds = type_casted_binds(binds)
-          log(sql, name, binds, type_casted_binds) do
+          typed_binds = type_casted_binds(binds)
+          log(sql, name, binds, typed_binds) do
             ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
-              @connection.exec_params(sql, type_casted_binds)
+              @connection.exec_params(prepare_statement(sql, typed_binds), [])
             end
           end
-        end
-
-        def exec_cache(sql, name, binds)
-          materialize_transactions
-          update_typemap_for_default_timezone
-
-          stmt_key = prepare_statement(sql, binds)
-          type_casted_binds = type_casted_binds(binds)
-
-          log(sql, name, binds, type_casted_binds, stmt_key) do
-            ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
-              @connection.exec_prepared(stmt_key, type_casted_binds)
-            end
-          end
-        rescue ActiveRecord::StatementInvalid => e
-          raise unless is_cached_plan_failure?(e)
-
-          # Nothing we can do if we are in a transaction because all commands
-          # will raise InFailedSQLTransaction
-          if in_transaction?
-            raise ActiveRecord::PreparedStatementCacheExpired.new(e.cause.message)
+        rescue ActiveRecord::StatementInvalid,
+               PG::InternalError => error
+          if error.message.include? "At least one input has no complete timestamps yet"
+            raise ::Materialize::Errors::IncompleteInput, error.message
           else
-            @lock.synchronize do
-              # outside of transactions we can simply flush this query and retry
-              @statements.delete sql_key(sql)
-            end
-            retry
+            raise
           end
         end
 
@@ -730,27 +575,33 @@ module ActiveRecord
         # Returns the statement identifier for the client side cache
         # of statements
         def sql_key(sql)
-          "#{schema_search_path}-#{sql}"
+          "#{sql}"
         end
 
-        # Prepare the statement if it hasn't been prepared, return
-        # the statement key.
         def prepare_statement(sql, binds)
-          @lock.synchronize do
-            sql_key = sql_key(sql)
-            unless @statements.key? sql_key
-              nextkey = @statements.next_key
-              begin
-                @connection.prepare nextkey, sql
-              rescue => e
-                raise translate_exception_class(e, sql, binds)
+          parametrized_statement = sql.split(/\s+/)
+          binds.each_with_index do |n, i|
+            index = i + 1
+            part = \
+              case n
+              when String
+                "'#{quote_string(n)}'"
+              when Numeric
+                n.to_s
               end
-              # Clear the queue
-              @connection.get_last_result
-              @statements[sql_key] = nextkey
+
+            parametrized_statement = parametrized_statement.map do |token|
+              token
+                .gsub("$#{index}", part)
+                .gsub("($#{index}", "(#{part}")
+                .gsub("$#{index})", "#{part})")
+                .gsub("($#{index})", "(#{part})")
+                .gsub(",$#{index}", ",#{part}")
+                .gsub("$#{index},", "#{part},")
+                .gsub(",$#{index},", ",#{part},")
             end
-            @statements[sql_key]
           end
+          parametrized_statement.join ' '
         end
 
         # Connects to a Materialize server and sets up the adapter depending on the
@@ -768,7 +619,6 @@ module ActiveRecord
           if @config[:encoding]
             @connection.set_client_encoding(@config[:encoding])
           end
-          self.schema_search_path = @config[:schema_search_path] || @config[:schema_order]
 
           variables = @config.fetch(:variables, {}).stringify_keys
 
@@ -814,16 +664,20 @@ module ActiveRecord
         #  - ::regclass is a function that gives the id for a table name
         def column_definitions(table_name)
           query(<<~SQL, "SCHEMA")
-              SELECT a.attname, format_type(a.atttypid, a.atttypmod),
-                     pg_get_expr(d.adbin, d.adrelid), a.attnotnull, a.atttypid, a.atttypmod,
-                     c.collname, col_description(a.attrelid, a.attnum) AS comment
-                FROM pg_attribute a
-                LEFT JOIN pg_attrdef d ON a.attrelid = d.adrelid AND a.attnum = d.adnum
-                LEFT JOIN pg_type t ON a.atttypid = t.oid
-                LEFT JOIN pg_collation c ON a.attcollation = c.oid AND a.attcollation <> t.typcollation
-               WHERE a.attrelid = #{quote(quote_table_name(table_name))}::regclass
-                 AND a.attnum > 0 AND NOT a.attisdropped
-               ORDER BY a.attnum
+          SELECT
+            c.name,
+            c.type AS format_type,
+            c.nullable,
+            NULL AS default,
+            NULL AS comment
+          FROM mz_columns c
+          LEFT JOIN mz_tables t ON c.id = t.id
+          LEFT JOIN mz_schemas s ON t.schema_id = s.id
+          LEFT JOIN mz_databases d ON s.database_id = d.id
+          WHERE
+            t.name = #{quote(table_name)}
+            AND d.name = #{quote(current_database)}
+            ORDER BY c.position
           SQL
         end
 
@@ -890,10 +744,15 @@ module ActiveRecord
             "int2" => PG::TextDecoder::Integer,
             "int4" => PG::TextDecoder::Integer,
             "int8" => PG::TextDecoder::Integer,
+            "integer" => PG::TextDecoder::Integer,
+            "bigint" => PG::TextDecoder::Integer,
             "oid" => PG::TextDecoder::Integer,
+            "float" => PG::TextDecoder::Float,
             "float4" => PG::TextDecoder::Float,
             "float8" => PG::TextDecoder::Float,
+            "double" => PG::TextDecoder::Float,
             "bool" => PG::TextDecoder::Boolean,
+            "boolean" => PG::TextDecoder::Boolean,
           }
 
           if defined?(PG::TextDecoder::TimestampUtc)
@@ -929,24 +788,12 @@ module ActiveRecord
         end
 
         ActiveRecord::Type.add_modifier({ array: true }, OID::Array, adapter: :materialize)
-        ActiveRecord::Type.add_modifier({ range: true }, OID::Range, adapter: :materialize)
-        ActiveRecord::Type.register(:bit, OID::Bit, adapter: :materialize)
-        ActiveRecord::Type.register(:bit_varying, OID::BitVarying, adapter: :materialize)
         ActiveRecord::Type.register(:binary, OID::Bytea, adapter: :materialize)
-        ActiveRecord::Type.register(:cidr, OID::Cidr, adapter: :materialize)
         ActiveRecord::Type.register(:date, OID::Date, adapter: :materialize)
         ActiveRecord::Type.register(:datetime, OID::DateTime, adapter: :materialize)
         ActiveRecord::Type.register(:decimal, OID::Decimal, adapter: :materialize)
         ActiveRecord::Type.register(:enum, OID::Enum, adapter: :materialize)
-        ActiveRecord::Type.register(:hstore, OID::Hstore, adapter: :materialize)
-        ActiveRecord::Type.register(:inet, OID::Inet, adapter: :materialize)
         ActiveRecord::Type.register(:jsonb, OID::Jsonb, adapter: :materialize)
-        ActiveRecord::Type.register(:money, OID::Money, adapter: :materialize)
-        ActiveRecord::Type.register(:point, OID::Point, adapter: :materialize)
-        ActiveRecord::Type.register(:legacy_point, OID::LegacyPoint, adapter: :materialize)
-        ActiveRecord::Type.register(:uuid, OID::Uuid, adapter: :materialize)
-        ActiveRecord::Type.register(:vector, OID::Vector, adapter: :materialize)
-        ActiveRecord::Type.register(:xml, OID::Xml, adapter: :materialize)
     end
   end
 end