activerecord-materialize-adapter 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1003f4f98f3745e7b60ae1784af47747ad13dcaa0dbc2dfc8fb96489e6a90f4
-  data.tar.gz: 07bfb83d66eea02e86e053a0be1ea88fb18a7a101a4ac89047889a098f2246d0
+  metadata.gz: 03b754e13f63726e0e11d14107d145449521404b78ccafb431d83de73dcf2360
+  data.tar.gz: bc53fd81cece00b418ea20ab4068bc5b063bee235f48bcd16f665d7be05ab4f7
 SHA512:
-  metadata.gz: 61b3e6efea873654b7a8225aed6062c3d87f8b78be411801e3fa9b25ed4a826bcce5a441c95caadf6ec4eb9d8792708e0e1cea56589e5e206c49d82b6eea15ff
-  data.tar.gz: 55bb91241a81f26ac8a3d2e662962c4195ba2517494fda6cd59549be030a3ee6d72137e7da933d463574cd314e88d66eeffd2445546b1cdf5f79cd4fafd3c42c
+  metadata.gz: 99041a07b198e3cd30000855d02698fd3e222040f2d25693060e407b9f068242d694785feecf2bb61187a72f30f02446f50e8da0fac2dbc7419ca9228b914523
+  data.tar.gz: dc272b7bbbd43a0ead177606b944b871f94fe8e85597c4e12263f1c987710b0bfd07d1030ea39581fcb7da92457edd6ea7db0cef3bf4700c3508b37945db0e61

data/lib/active_record/connection_adapters/materialize/column.rb

@@ -6,15 +6,6 @@ module ActiveRecord
       class Column < ActiveRecord::ConnectionAdapters::Column # :nodoc:
         delegate :oid, :fmod, to: :sql_type_metadata
 
-        def initialize(*, serial: nil, **)
-          super
-          @serial = serial
-        end
-
-        def serial?
-          @serial
-        end
-
         def array
           sql_type_metadata.sql_type.end_with?("[]")
         end

data/lib/active_record/connection_adapters/materialize/database_statements.rb

@@ -11,9 +11,9 @@ module ActiveRecord
           Materialize::ExplainPrettyPrinter.new.pp(exec_query(sql, "EXPLAIN", binds))
         end
 
-        # The internal Materialize identifier of the money data type.
+        # The internal PostgreSQL identifier of the money data type.
         MONEY_COLUMN_TYPE_OID = 790 #:nodoc:
-        # The internal Materialize identifier of the BYTEA data type.
+        # The internal PostgreSQL identifier of the BYTEA data type.
         BYTEA_COLUMN_TYPE_OID = 17 #:nodoc:
 
         # create a 2D array representing the result set
@@ -62,15 +62,17 @@ module ActiveRecord
         def query(sql, name = nil) #:nodoc:
           materialize_transactions
 
+          result = nil
           log(sql, name) do
             ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
-              result_as_array @connection.async_exec(sql)
+              result = execute_async_and_raise(sql)
             end
           end
+          result_as_array result
         end
 
         READ_QUERY = ActiveRecord::ConnectionAdapters::AbstractAdapter.build_read_query_regexp(
-          :begin, :commit, :explain, :select, :set, :show, :release, :savepoint, :rollback, :with
+          :begin, :commit, :explain, :select, :set, :show, :rollback, :with
         ) # :nodoc:
         private_constant :READ_QUERY
 
@@ -89,20 +91,13 @@ module ActiveRecord
 
           materialize_transactions
 
+          result = nil
           log(sql, name) do
             ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
-              @connection.async_exec(sql)
+              result = execute_async_and_raise(sql)
             end
           end
-
-        # Known issue: PG::InternalError: ERROR:  At least one input has no complete timestamps yet
-        # https://github.com/MaterializeInc/materialize/issues/2917
-        rescue ActiveRecord::StatementInvalid => error
-          if error.message.include? "At least one input has no complete timestamps yet"
-            raise ::Materialize::Errors::IncompleteInput, error.message
-          else
-            raise
-          end
+          result
         end
 
         def exec_query(sql, name = "SQL", binds = [], prepare: false)
@@ -112,6 +107,7 @@ module ActiveRecord
             fields.each_with_index do |fname, i|
               ftype = result.ftype i
               fmod  = result.fmod i
+
               types[fname] = get_oid_type(ftype, fmod, fname)
             end
             ActiveRecord::Result.new(fields, result.values, types)
@@ -130,32 +126,10 @@ module ActiveRecord
             pk = primary_key(table_ref) if table_ref
           end
 
-          if pk = suppress_composite_primary_key(pk)
-            sql = "#{sql} RETURNING #{quote_column_name(pk)}"
-          end
-
           super
         end
         private :sql_for_insert
 
-        def exec_insert(sql, name = nil, binds = [], pk = nil, sequence_name = nil)
-          if use_insert_returning? || pk == false
-            super
-          else
-            result = exec_query(sql, name, binds)
-            unless sequence_name
-              table_ref = extract_table_ref_from_insert_sql(sql)
-              if table_ref
-                pk = primary_key(table_ref) if pk.nil?
-                pk = suppress_composite_primary_key(pk)
-                sequence_name = default_sequence_name(table_ref, pk)
-              end
-              return result unless sequence_name
-            end
-            last_insert_id_result(sequence_name)
-          end
-        end
-
         # Begins a transaction.
         def begin_db_transaction
           execute "BEGIN"
@@ -177,6 +151,18 @@ module ActiveRecord
         end
 
         private
+          # Known issue: PG::InternalError: ERROR:  At least one input has no complete timestamps yet
+          # https://github.com/MaterializeInc/materialize/issues/2917
+          def execute_async_and_raise(sql)
+            @connection.async_exec(sql)
+          rescue PG::InternalError => error
+            if error.message.include? "At least one input has no complete timestamps yet"
+              raise ::Materialize::Errors::IncompleteInput, error.message
+            else
+              raise
+            end
+          end
+
           def execute_batch(statements, name = nil)
             execute(combine_multi_statements(statements))
           end
@@ -185,11 +171,6 @@ module ActiveRecord
             ["TRUNCATE TABLE #{table_names.map(&method(:quote_table_name)).join(", ")}"]
           end
 
-          # Returns the current ID of a table's sequence.
-          def last_insert_id_result(sequence_name)
-            exec_query("SELECT currval(#{quote(sequence_name)})", "SQL")
-          end
-
           def suppress_composite_primary_key(pk)
             pk unless pk.is_a?(Array)
           end

data/lib/active_record/connection_adapters/materialize/oid.rb

@@ -5,24 +5,12 @@ require "active_record/connection_adapters/materialize/oid/array"
 require "active_record/connection_adapters/materialize/oid/bit"
 require "active_record/connection_adapters/materialize/oid/bit_varying"
 require "active_record/connection_adapters/materialize/oid/bytea"
-require "active_record/connection_adapters/materialize/oid/cidr"
 require "active_record/connection_adapters/materialize/oid/date"
 require "active_record/connection_adapters/materialize/oid/date_time"
 require "active_record/connection_adapters/materialize/oid/decimal"
 require "active_record/connection_adapters/materialize/oid/enum"
-require "active_record/connection_adapters/materialize/oid/hstore"
-require "active_record/connection_adapters/materialize/oid/inet"
 require "active_record/connection_adapters/materialize/oid/jsonb"
-require "active_record/connection_adapters/materialize/oid/money"
 require "active_record/connection_adapters/materialize/oid/oid"
-require "active_record/connection_adapters/materialize/oid/point"
-require "active_record/connection_adapters/materialize/oid/legacy_point"
-require "active_record/connection_adapters/materialize/oid/range"
-require "active_record/connection_adapters/materialize/oid/specialized_string"
-require "active_record/connection_adapters/materialize/oid/uuid"
-require "active_record/connection_adapters/materialize/oid/vector"
-require "active_record/connection_adapters/materialize/oid/xml"
-
 require "active_record/connection_adapters/materialize/oid/type_map_initializer"
 
 module ActiveRecord

data/lib/active_record/connection_adapters/materialize/oid/type_map_initializer.rb

@@ -6,7 +6,7 @@ module ActiveRecord
   module ConnectionAdapters
     module Materialize
       module OID # :nodoc:
-        # This class uses the data from Materialize pg_type table to build
+        # This class uses the data from Materialize mz_types table to build
         # the OID -> Type mapping.
         #   - OID is an integer representing the type.
         #   - Type is an OID::Type object.
@@ -18,65 +18,22 @@ module ActiveRecord
 
           def run(records)
             nodes = records.reject { |row| @store.key? row["oid"].to_i }
-            mapped = nodes.extract! { |row| @store.key? row["typname"] }
-            ranges = nodes.extract! { |row| row["typtype"] == "r" }
-            enums = nodes.extract! { |row| row["typtype"] == "e" }
-            domains = nodes.extract! { |row| row["typtype"] == "d" }
-            arrays = nodes.extract! { |row| row["typinput"] == "array_in" }
-            composites = nodes.extract! { |row| row["typelem"].to_i != 0 }
+            mapped = nodes.extract! { |row| @store.key? row["name"] }
 
-            mapped.each     { |row| register_mapped_type(row)    }
-            enums.each      { |row| register_enum_type(row)      }
-            domains.each    { |row| register_domain_type(row)    }
-            arrays.each     { |row| register_array_type(row)     }
-            ranges.each     { |row| register_range_type(row)     }
-            composites.each { |row| register_composite_type(row) }
+            mapped.each { |row| register_mapped_type(row) }
           end
 
           def query_conditions_for_initial_load
             known_type_names = @store.keys.map { |n| "'#{n}'" }
             known_type_types = %w('r' 'e' 'd')
             <<~SQL % [known_type_names.join(", "), known_type_types.join(", ")]
-              WHERE
-                t.typname IN (%s)
-                OR t.typtype IN (%s)
-                OR t.typelem != 0
+              AND t.name IN (%s)
             SQL
           end
 
           private
             def register_mapped_type(row)
-              alias_type row["oid"], row["typname"]
-            end
-
-            def register_enum_type(row)
-              register row["oid"], OID::Enum.new
-            end
-
-            def register_array_type(row)
-              register_with_subtype(row["oid"], row["typelem"].to_i) do |subtype|
-                OID::Array.new(subtype, row["typdelim"])
-              end
-            end
-
-            def register_range_type(row)
-              register_with_subtype(row["oid"], row["rngsubtype"].to_i) do |subtype|
-                OID::Range.new(subtype, row["typname"].to_sym)
-              end
-            end
-
-            def register_domain_type(row)
-              if base_type = @store.lookup(row["typbasetype"].to_i)
-                register row["oid"], base_type
-              else
-                warn "unknown base type (OID: #{row["typbasetype"]}) for domain #{row["typname"]}."
-              end
-            end
-
-            def register_composite_type(row)
-              if subtype = @store.lookup(row["typelem"].to_i)
-                register row["oid"], OID::Vector.new(row["typdelim"], subtype)
-              end
+              alias_type row["oid"], row["name"]
             end
 
             def register(oid, oid_type = nil, &block)

data/lib/active_record/connection_adapters/materialize/quoting.rb

@@ -116,14 +116,8 @@ module ActiveRecord
         private_constant :COLUMN_NAME, :COLUMN_NAME_WITH_ORDER
 
         private
-          def lookup_cast_type(sql_type)
-            super(query_value("SELECT #{quote(sql_type)}::regtype::oid", "SCHEMA").to_i)
-          end
-
           def _quote(value)
             case value
-            when OID::Xml::Data
-              "xml '#{quote_string(value.to_s)}'"
             when OID::Bit::Data
               if value.binary?
                 "B'#{value}'"
@@ -152,7 +146,7 @@ module ActiveRecord
               # See https://deveiate.org/code/pg/PG/Connection.html#method-i-exec_prepared-doc
               # for more information
               { value: value.to_s, format: 1 }
-            when OID::Xml::Data, OID::Bit::Data
+            when OID::Bit::Data
               value.to_s
             when OID::Array::Data
               encode_array(value)

data/lib/active_record/connection_adapters/materialize/schema/source_statements.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Materialize
+      module Schema
+        module SourceStatements
+
+          def sources
+            query_values(data_source_sql(type: "SOURCE"), "SCHEMA")
+          end
+
+          # Get source options from an existing source
+          def source_options(source_name)
+            name_ref, statement = query("SHOW CREATE SOURCE #{quote_table_name(source_name)}", "SCHEMA").first
+            database_name, schema_name, publication_name = name_ref.split(".")
+            materialized = !!/CREATE\sMATERIALIZED/.match(statement)
+             _, source_type, _ = /FROM\s(POSTGRES|KAFKA\sBROKER)\sCONNECTION/.match(statement).to_s.split " "
+             source_type = source_type.to_s.downcase.to_sym
+
+            {
+              database_name: database_name,
+              schema_name: schema_name,
+              source_name: source_name,
+              source_type: source_type,
+              publication: publication_name,
+              materialized: materialized || source_type == :postgres
+            }
+          end
+
+          # "host=postgresdb port=5432 user=postgres dbname=source_database"
+          def select_database_config(connection_params)
+            {
+              host: connection_params['host'],
+              port: connection_params['port'],
+              user: connection_params['username'],
+              dbname: connection_params['database'],
+              password: connection_params['password']
+            }.compact
+          end
+
+          def create_source(source_name, publication:, source_type: :postgres, materialized: true, connection_params: nil)
+            materialized_statement = materialized ? 'MATERIALIZED ' : ''
+            connection_string = select_database_config(connection_params).map { |k, v| [k, v].join("=") }.join(" ")
+            case source_type
+            when :postgres
+              execute <<-SQL.squish
+                CREATE #{materialized_statement}SOURCE #{quote_schema_name(source_name)} FROM POSTGRES
+                  CONNECTION #{quote(connection_string)}
+                  PUBLICATION #{quote(publication)}
+              SQL
+            else
+              raise "Source type #{source_type} is currently unsupported for Materialized sources", NotImplementedError
+            end
+          end
+
+          def drop_source(source_name)
+          end
+
+          def source_exists?(source_name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/materialize/schema/view_statements.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Materialize
+      module Schema
+        module ViewStatements
+
+          # Returns an array of view names defined in the database.
+          def materialized_views
+            query_values(data_source_sql(type: "MATERIALIZED"), "SCHEMA")
+          end
+
+          def view_sql(view_name)
+            query("SHOW CREATE VIEW #{quote_table_name(view_name)}", "SCHEMA").first.last
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/materialize/schema_creation.rb

@@ -5,71 +5,36 @@ module ActiveRecord
     module Materialize
       class SchemaCreation < AbstractAdapter::SchemaCreation # :nodoc:
         private
-          def visit_AlterTable(o)
-            super << o.constraint_validations.map { |fk| visit_ValidateConstraint fk }.join(" ")
-          end
-
-          def visit_AddForeignKey(o)
-            super.dup.tap { |sql| sql << " NOT VALID" unless o.validate? }
-          end
-
-          def visit_ValidateConstraint(name)
-            "VALIDATE CONSTRAINT #{quote_column_name(name)}"
-          end
-
-          def visit_ChangeColumnDefinition(o)
-            column = o.column
-            column.sql_type = type_to_sql(column.type, **column.options)
-            quoted_column_name = quote_column_name(o.name)
-
-            change_column_sql = +"ALTER COLUMN #{quoted_column_name} TYPE #{column.sql_type}"
+          def visit_TableDefinition(o)
+            create_sql = +"CREATE#{table_modifier_in_create(o)} TABLE "
+            create_sql << "IF NOT EXISTS " if o.if_not_exists
+            create_sql << "#{quote_table_name(o.name)} "
 
-            options = column_options(column)
+            statements = o.columns.map { |c| accept c }
 
-            if options[:collation]
-              change_column_sql << " COLLATE \"#{options[:collation]}\""
+            if supports_indexes_in_create?
+              statements.concat(o.indexes.map { |column_name, options| index_in_create(o.name, column_name, options) })
             end
 
-            if options[:using]
-              change_column_sql << " USING #{options[:using]}"
-            elsif options[:cast_as]
-              cast_as_type = type_to_sql(options[:cast_as], **options)
-              change_column_sql << " USING CAST(#{quoted_column_name} AS #{cast_as_type})"
+            if supports_foreign_keys?
+              statements.concat(o.foreign_keys.map { |to_table, options| foreign_key_in_create(o.name, to_table, options) })
             end
 
-            if options.key?(:default)
-              if options[:default].nil?
-                change_column_sql << ", ALTER COLUMN #{quoted_column_name} DROP DEFAULT"
-              else
-                quoted_default = quote_default_expression(options[:default], column)
-                change_column_sql << ", ALTER COLUMN #{quoted_column_name} SET DEFAULT #{quoted_default}"
-              end
-            end
-
-            if options.key?(:null)
-              change_column_sql << ", ALTER COLUMN #{quoted_column_name} #{options[:null] ? 'DROP' : 'SET'} NOT NULL"
-            end
-
-            change_column_sql
+            create_sql << "(#{statements.join(', ')})" if statements.present?
+            add_table_options!(create_sql, table_options(o))
+            create_sql << " AS #{to_sql(o.as)}" if o.as
+            create_sql
           end
 
           def add_column_options!(sql, options)
-            if options[:collation]
-              sql << " COLLATE \"#{options[:collation]}\""
+            sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options)
+            # must explicitly check for :null to allow change_column to work on migrations
+            if options[:null] == false
+              sql << " NOT NULL"
             end
-            super
+            sql
           end
 
-          # Returns any SQL string to go between CREATE and TABLE. May be nil.
-          def table_modifier_in_create(o)
-            # A table cannot be both TEMPORARY and UNLOGGED, since all TEMPORARY
-            # tables are already UNLOGGED.
-            if o.temporary
-              " TEMPORARY"
-            elsif o.unlogged
-              " UNLOGGED"
-            end
-          end
       end
     end
   end