activerecord-materialized 0.1.0 → 0.2.0

data/lib/generators/activerecord_materialized/install/templates/create_ar_materialized_view_metadata.rb.erb

@@ -12,6 +12,9 @@ class CreateArMaterializedViewMetadata < ActiveRecord::Migration[<%= ActiveRecor
       t.integer :refresh_duration_ms
       t.text :last_error
       t.text :maintenance_payload
+      t.datetime :last_reconciled_at
+      t.integer :reconciled_partition_count
+      t.integer :fresh_set_generation, null: false, default: 0
 
       t.timestamps
     end
@@ -21,10 +24,21 @@ class CreateArMaterializedViewMetadata < ActiveRecord::Migration[<%= ActiveRecor
     create_table :ar_materialized_view_partitions do |t|
       t.string :view_name, null: false
       t.string :partition_key, null: false
+      t.integer :generation, null: false, default: 0
 
       t.datetime :created_at, null: false
     end
 
     add_index :ar_materialized_view_partitions, %i[view_name partition_key], unique: true
+
+    create_table :ar_materialized_view_source_watermarks do |t|
+      t.string :view_name, null: false
+      t.string :partition_key, null: false
+      t.bigint :source_ts, null: false
+
+      t.timestamps
+    end
+
+    add_index :ar_materialized_view_source_watermarks, %i[view_name partition_key], unique: true
   end
 end

data/lib/generators/activerecord_materialized/install_generator.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 require "rails/generators"
@@ -7,24 +6,20 @@ require "rails/generators/migration"
 module ActiverecordMaterialized
   # `rails generate activerecord_materialized:install` — installs the metadata-table migration.
   class InstallGenerator < ::Rails::Generators::Base
-    extend T::Sig
     include Rails::Generators::Migration
 
     source_root File.expand_path("install", __dir__)
 
-    sig { params(dirname: String).returns(String) }
     def self.next_migration_number(dirname)
-      next_migration_number = T.unsafe(self).current_migration_number(dirname) + 1
+      next_migration_number = current_migration_number(dirname) + 1
       ::ActiveRecord::Migration.next_migration_number(next_migration_number)
     end
 
-    sig { void }
     def copy_migration
       migration_template "create_ar_materialized_view_metadata.rb.erb",
                          "db/migrate/create_ar_materialized_view_metadata.rb"
     end
 
-    sig { void }
     def show_readme
       readme "README" if behavior == :invoke
     end

data/lib/generators/activerecord_materialized/migration_generator.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 require "rails/generators"
@@ -12,17 +11,14 @@ module ActiverecordMaterialized
   #
   # Run after the view's `materialized_from` is defined, then `db:migrate`.
   class MigrationGenerator < ::Rails::Generators::NamedBase
-    extend T::Sig
     include ::Rails::Generators::Migration
 
     source_root File.expand_path("templates", __dir__)
 
-    sig { params(dirname: String).returns(String) }
     def self.next_migration_number(dirname)
-      ::ActiveRecord::Migration.next_migration_number(T.unsafe(self).current_migration_number(dirname) + 1)
+      ::ActiveRecord::Migration.next_migration_number(current_migration_number(dirname) + 1)
     end
 
-    sig { void }
     def create_migration_file
       migration_template "materialized_view_migration.rb.erb",
                          File.join("db", "migrate", "create_#{builder.table_name}.rb")
@@ -30,15 +26,13 @@ module ActiverecordMaterialized
 
     private
 
-    sig { returns(::ActiveRecord::Materialized::MigrationBuilder) }
     def builder
-      @builder ||= T.let(build_builder, T.nilable(::ActiveRecord::Materialized::MigrationBuilder))
+      @builder ||= build_builder
     end
 
     # Resolve via the registry (after eager-load) rather than constantize.
-    sig { returns(::ActiveRecord::Materialized::MigrationBuilder) }
     def build_builder
-      T.unsafe(::Rails).application.eager_load!
+      ::Rails.application.eager_load!
       view_class = ::ActiveRecord::Materialized::Registry.for_class_name(class_name)
       if view_class.nil?
         Kernel.raise ::Thor::Error,

data/lib/generators/activerecord_materialized/outbox_generator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/migration"
+
+module ActiverecordMaterialized
+  # Generates a migration that installs write-outbox triggers on a dependency table, so writes that
+  # bypass ActiveRecord (raw SQL, another service, a backfill) are still captured and can be relayed
+  # into view maintenance via +ActiveRecord::Materialized.drain_write_outbox!+:
+  #
+  #   bin/rails generate activerecord_materialized:outbox line_items category region
+  #
+  # The trailing arguments are the GROUP BY key columns to capture (empty for an un-grouped view).
+  # The migration emits the correct trigger DDL for the connection's adapter at migrate time.
+  # See +docs/out-of-band-writes.md+.
+  class OutboxGenerator < ::Rails::Generators::NamedBase
+    include ::Rails::Generators::Migration
+
+    source_root File.expand_path("templates", __dir__)
+
+    argument :columns, type: :array, default: [],
+                       banner: "key_column key_column",
+                       desc: "GROUP BY key columns to capture (the partition keys)"
+
+    def self.next_migration_number(dirname)
+      ::ActiveRecord::Migration.next_migration_number(current_migration_number(dirname) + 1)
+    end
+
+    def create_migration_file
+      # +file_name+, not NamedBase#table_name: NAME is a raw dependency table, and table_name would
+      # inflect it (pluralize a singular/irregular table — person -> people), targeting the wrong table.
+      migration_template "write_outbox_migration.rb.erb",
+                         File.join("db", "migrate", "install_write_outbox_triggers_on_#{file_name}.rb")
+    end
+  end
+end

data/lib/generators/activerecord_materialized/templates/materialized_view_migration.rb.erb

@@ -4,7 +4,11 @@ class <%= builder.migration_class_name %> < ActiveRecord::Migration[<%= builder.
   def change
     create_table :<%= builder.table_name %> do |t|
 <% builder.column_definitions.each do |column| -%>
-      t.<%= column.type %> :<%= column.name %>
+      t.<%= column.type %> :<%= column.name %><%= column.migration_arguments %>
+<% end -%>
+<% if (index = builder.index_definition) -%>
+      # Incremental maintenance filters and deletes by the GROUP BY key; this index keeps it partition-local.
+      t.index <%= index.columns.map(&:to_sym).inspect %><%= ", unique: true" if index.unique %>
 <% end -%>
     end
   end

data/lib/generators/activerecord_materialized/templates/write_outbox_migration.rb.erb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Installs activerecord-materialized write-outbox triggers on <%= table_name %>. The triggers append
+# every INSERT/UPDATE/DELETE — including raw SQL and writes from other services — to the outbox table,
+# so out-of-band writes still reach view maintenance. Relay them with
+# ActiveRecord::Materialized.drain_write_outbox! from a poller, cron, or background job.
+#
+# The correct trigger DDL for this database's adapter (SQLite/MySQL/Postgres) is emitted at migrate
+# time; the outbox table is provisioned lazily on first install. See docs/out-of-band-writes.md.
+class InstallWriteOutboxTriggersOn<%= file_name.camelize %> < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def up
+    ActiveRecord::Materialized::WriteOutbox.install_triggers!(
+      :<%= file_name %>,
+      key_columns: %i[<%= columns.join(" ") %>]
+    )
+  end
+
+  def down
+    ActiveRecord::Materialized::WriteOutbox.uninstall_triggers!(:<%= file_name %>)
+  end
+end

data/lib/generators/activerecord_materialized/view_generator.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 require "rails/generators"
@@ -6,11 +5,8 @@ require "rails/generators"
 module ActiverecordMaterialized
   # `rails generate activerecord_materialized:view NAME` — scaffolds a materialized view class.
   class ViewGenerator < ::Rails::Generators::NamedBase
-    extend T::Sig
-
     source_root File.expand_path("templates", __dir__)
 
-    sig { void }
     def create_model
       template "materialized_view.rb.erb", File.join("app/models", class_path, "#{file_name}.rb")
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-materialized
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Michael Avrukin
@@ -51,20 +51,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '8.0'
-- !ruby/object:Gem::Dependency
-  name: sorbet-runtime
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.5'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.5'
 description: |
   Provides transparent materialized view semantics for Rails applications
   on databases that do not support native materialized views. Precomputes
@@ -82,26 +68,43 @@ files:
 - lib/activerecord/materialized/aggregate_analysis.rb
 - lib/activerecord/materialized/async_refresher.rb
 - lib/activerecord/materialized/cache_table_schema.rb
+- lib/activerecord/materialized/change_source.rb
 - lib/activerecord/materialized/cold_read.rb
+- lib/activerecord/materialized/column_type_inference.rb
+- lib/activerecord/materialized/configurable_table_name.rb
 - lib/activerecord/materialized/configuration.rb
+- lib/activerecord/materialized/connection_routing.rb
+- lib/activerecord/materialized/data_verification_result.rb
+- lib/activerecord/materialized/data_verifier.rb
+- lib/activerecord/materialized/debezium_envelope.rb
 - lib/activerecord/materialized/delta_maintainer.rb
 - lib/activerecord/materialized/dependency_registry.rb
 - lib/activerecord/materialized/dependency_trackable.rb
 - lib/activerecord/materialized/incremental_maintainer.rb
+- lib/activerecord/materialized/incremental_refresh.rb
+- lib/activerecord/materialized/instrumentation.rb
 - lib/activerecord/materialized/maintenance_delta.rb
 - lib/activerecord/materialized/maintenance_delta_builder.rb
 - lib/activerecord/materialized/maintenance_store.rb
 - lib/activerecord/materialized/metadata.rb
 - lib/activerecord/materialized/metadata/maintenance_payload.rb
+- lib/activerecord/materialized/metadata/reconciliation.rb
 - lib/activerecord/materialized/metadata/schema.rb
 - lib/activerecord/materialized/metadata/timestamps.rb
 - lib/activerecord/materialized/metadata_record.rb
 - lib/activerecord/materialized/migration_builder.rb
 - lib/activerecord/materialized/module_api.rb
+- lib/activerecord/materialized/partition_filter.rb
+- lib/activerecord/materialized/partition_keyed_store.rb
 - lib/activerecord/materialized/partition_record.rb
+- lib/activerecord/materialized/partition_snapshot.rb
 - lib/activerecord/materialized/partition_state.rb
 - lib/activerecord/materialized/query_expressions.rb
 - lib/activerecord/materialized/railtie.rb
+- lib/activerecord/materialized/read_router.rb
+- lib/activerecord/materialized/reconcile_job.rb
+- lib/activerecord/materialized/reconcile_result.rb
+- lib/activerecord/materialized/reconciler.rb
 - lib/activerecord/materialized/refresh_callbacks.rb
 - lib/activerecord/materialized/refresh_job.rb
 - lib/activerecord/materialized/refresh_result.rb
@@ -110,27 +113,33 @@ files:
 - lib/activerecord/materialized/registry.rb
 - lib/activerecord/materialized/relation_cache_writer.rb
 - lib/activerecord/materialized/schema_verifier.rb
+- lib/activerecord/materialized/source_watermark.rb
+- lib/activerecord/materialized/source_watermark_record.rb
 - lib/activerecord/materialized/summary_delta.rb
 - lib/activerecord/materialized/summary_delta_builder.rb
 - lib/activerecord/materialized/table_model_registry.rb
+- lib/activerecord/materialized/table_swap.rb
 - lib/activerecord/materialized/tasks.rb
-- lib/activerecord/materialized/type_reexports.rb
 - lib/activerecord/materialized/version.rb
 - lib/activerecord/materialized/view.rb
-- lib/activerecord/materialized/view_class.rb
 - lib/activerecord/materialized/view_configuration_class_methods.rb
 - lib/activerecord/materialized/view_definition.rb
 - lib/activerecord/materialized/view_incremental_class_methods.rb
+- lib/activerecord/materialized/view_loader.rb
 - lib/activerecord/materialized/view_query_access_class_methods.rb
 - lib/activerecord/materialized/view_refresh_policy_class_methods.rb
 - lib/activerecord/materialized/write_change.rb
-- lib/activerecord_materialized_types.rb
+- lib/activerecord/materialized/write_maintenance.rb
+- lib/activerecord/materialized/write_outbox.rb
+- lib/activerecord/materialized/write_outbox_record.rb
 - lib/generators/activerecord_materialized/install/templates/README
 - lib/generators/activerecord_materialized/install/templates/create_ar_materialized_view_metadata.rb.erb
 - lib/generators/activerecord_materialized/install_generator.rb
 - lib/generators/activerecord_materialized/migration_generator.rb
+- lib/generators/activerecord_materialized/outbox_generator.rb
 - lib/generators/activerecord_materialized/templates/materialized_view.rb.erb
 - lib/generators/activerecord_materialized/templates/materialized_view_migration.rb.erb
+- lib/generators/activerecord_materialized/templates/write_outbox_migration.rb.erb
 - lib/generators/activerecord_materialized/view_generator.rb
 homepage: https://github.com/mavrukin/activerecord-materialized
 licenses:
@@ -156,7 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.15
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Application-level materialized views for ActiveRecord
 test_files: []

data/lib/activerecord/materialized/type_reexports.rb

@@ -1,14 +0,0 @@
-# typed: strict
-# frozen_string_literal: true
-
-module ActiveRecord
-  module Materialized
-    DebounceInterval = T.type_alias { ::ActiveRecordMaterializedTypes::DebounceInterval }
-    StalenessDuration = T.type_alias { ::ActiveRecordMaterializedTypes::StalenessDuration }
-    SourceDefinition = T.type_alias { ::ActiveRecordMaterializedTypes::SourceDefinition }
-    RefreshMode = T.type_alias { ::ActiveRecordMaterializedTypes::RefreshMode }
-    RefreshCallbackName = T.type_alias { ::ActiveRecordMaterializedTypes::RefreshCallbackName }
-    Connection = T.type_alias { ::ActiveRecordMaterializedTypes::Connection }
-    Timestamp = T.type_alias { ::ActiveRecordMaterializedTypes::Timestamp }
-  end
-end

data/lib/activerecord/materialized/view_class.rb

@@ -1,8 +0,0 @@
-# typed: strict
-# frozen_string_literal: true
-
-module ActiveRecord
-  module Materialized
-    ViewClass = T.type_alias { T.class_of(View) }
-  end
-end

data/lib/activerecord_materialized_types.rb

@@ -1,18 +0,0 @@
-# typed: strict
-# frozen_string_literal: true
-
-module ActiveRecordMaterializedTypes
-  DebounceInterval = T.type_alias { T.any(Integer, Float, ::ActiveSupport::Duration) }
-  StalenessDuration = T.type_alias { T.any(Integer, ::ActiveSupport::Duration) }
-  SourceDefinition = T.type_alias do
-    T.any(
-      ::ActiveRecord::Relation,
-      Proc,
-      T.nilable(T.proc.returns(::ActiveRecord::Relation))
-    )
-  end
-  RefreshMode = T.type_alias { Symbol }
-  RefreshCallbackName = T.type_alias { T.any(Symbol, Proc) }
-  Connection = T.type_alias { ::ActiveRecord::ConnectionAdapters::AbstractAdapter }
-  Timestamp = T.type_alias { T.any(::Time, ::ActiveSupport::TimeWithZone) }
-end