activerecord-materialized 0.1.0

data/lib/activerecord/materialized/write_change.rb

@@ -0,0 +1,69 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # A committed write to a dependency table, captured as full before/after
+    # attribute snapshots (string-keyed). Snapshots are complete — not just the
+    # changed columns — so maintenance can compute deltas even when the GROUP BY
+    # key or a summed column did not change.
+    class WriteChange
+      extend T::Sig
+
+      Operation = T.type_alias { T.any(Symbol, String) }
+      Attributes = T.type_alias { T::Hash[String, T.untyped] }
+
+      sig { returns(String) }
+      attr_reader :table_name
+
+      sig { returns(Operation) }
+      attr_reader :operation
+
+      sig { returns(Attributes) }
+      attr_reader :before
+
+      sig { returns(Attributes) }
+      attr_reader :after
+
+      sig { params(table_name: String, operation: Operation, before: Attributes, after: Attributes).void }
+      def initialize(table_name:, operation:, before:, after:)
+        @table_name = table_name
+        @operation = operation
+        @before = before
+        @after = after
+      end
+
+      sig { params(record: ::ActiveRecord::Base, operation: Operation).returns(WriteChange) }
+      def self.from_record(record, operation)
+        table_name = record.class.table_name
+        case operation.to_sym
+        when :create
+          new(table_name: table_name, operation: :create, before: {}, after: stringify_keys(record.attributes))
+        when :update
+          new(table_name: table_name, operation: :update, before: old_attributes(record),
+              after: stringify_keys(record.attributes))
+        when :destroy
+          # attributes_in_database is emptied once the row is gone; use in-memory attributes.
+          new(table_name: table_name, operation: :destroy, before: stringify_keys(record.attributes), after: {})
+        else
+          raise ArgumentError, "unsupported write operation: #{operation}"
+        end
+      end
+
+      # Full pre-update attributes: current values with changed columns reverted.
+      sig { params(record: ::ActiveRecord::Base).returns(Attributes) }
+      def self.old_attributes(record)
+        attributes = stringify_keys(record.attributes)
+        record.saved_changes.each_pair { |column, (old_value, _new_value)| attributes[column.to_s] = old_value }
+        attributes
+      end
+
+      sig { params(values: T::Hash[T.any(String, Symbol), T.untyped]).returns(Attributes) }
+      def self.stringify_keys(values)
+        values.each_with_object({}) { |(key, value), result| result[key.to_s] = value }
+      end
+
+      private_class_method :old_attributes, :stringify_keys
+    end
+  end
+end

data/lib/activerecord/materialized.rb

@@ -0,0 +1,55 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+require "active_record"
+require "active_support/concern"
+require "active_support/core_ext/string/inflections"
+require "active_support/core_ext/module/delegation"
+require "active_support/core_ext/integer/time"
+require "active_support/core_ext/object/blank"
+require "securerandom"
+
+require_relative "../activerecord_materialized_types"
+require_relative "materialized/type_reexports"
+require_relative "materialized/configuration"
+require_relative "materialized/module_api"
+require_relative "materialized/refresh_callbacks"
+require_relative "materialized/write_change"
+require_relative "materialized/maintenance_delta_builder"
+require_relative "materialized/table_model_registry"
+require_relative "materialized/dependency_trackable"
+require_relative "materialized/dependency_registry"
+require_relative "materialized/async_refresher"
+require_relative "materialized/refresh_scheduler"
+require_relative "materialized/registry"
+require_relative "materialized/metadata_record"
+require_relative "materialized/view_incremental_class_methods"
+require_relative "materialized/view_refresh_policy_class_methods"
+require_relative "materialized/view_configuration_class_methods"
+require_relative "materialized/view_query_access_class_methods"
+require_relative "materialized/view"
+require_relative "materialized/view_class"
+require_relative "materialized/refresh_result"
+require_relative "materialized/refresher"
+require_relative "materialized/maintenance_delta"
+require_relative "materialized/view_definition"
+require_relative "materialized/aggregate_analysis"
+require_relative "materialized/summary_delta"
+require_relative "materialized/summary_delta_builder"
+require_relative "materialized/query_expressions"
+require_relative "materialized/cache_table_schema"
+require_relative "materialized/migration_builder"
+require_relative "materialized/schema_verifier"
+require_relative "materialized/cold_read"
+require_relative "materialized/relation_cache_writer"
+require_relative "materialized/maintenance_store"
+require_relative "materialized/incremental_maintainer"
+require_relative "materialized/delta_maintainer"
+require_relative "materialized/metadata"
+require_relative "materialized/partition_record"
+require_relative "materialized/partition_state"
+
+require_relative "materialized/refresh_job" if defined?(ActiveJob::Base)
+
+require_relative "materialized/railtie" if defined?(Rails::Railtie)

data/lib/activerecord_materialized_types.rb

@@ -0,0 +1,18 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecordMaterializedTypes
+  DebounceInterval = T.type_alias { T.any(Integer, Float, ::ActiveSupport::Duration) }
+  StalenessDuration = T.type_alias { T.any(Integer, ::ActiveSupport::Duration) }
+  SourceDefinition = T.type_alias do
+    T.any(
+      ::ActiveRecord::Relation,
+      Proc,
+      T.nilable(T.proc.returns(::ActiveRecord::Relation))
+    )
+  end
+  RefreshMode = T.type_alias { Symbol }
+  RefreshCallbackName = T.type_alias { T.any(Symbol, Proc) }
+  Connection = T.type_alias { ::ActiveRecord::ConnectionAdapters::AbstractAdapter }
+  Timestamp = T.type_alias { T.any(::Time, ::ActiveSupport::TimeWithZone) }
+end

data/lib/generators/activerecord_materialized/install/templates/README

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+After installing activerecord-materialized, create materialized views as standard
+ActiveRecord models inheriting from ActiveRecord::Materialized::View.
+
+Example:
+
+  class SalesSummary < ActiveRecord::Materialized::View
+    self.table_name = "mv_sales_summary"
+
+    materialized_from do
+      LineItem.group(:category).select(
+        LineItem.arel_table[:category],
+        ActiveRecord::Materialized::QueryExpressions.sum_as(LineItem.arel_table[:amount], as: :total)
+      )
+    end
+
+    depends_on :line_items
+    max_staleness 12.hours
+    # cold_read :read_through  # default; also :serve_stale or :raise
+    # warm_up { [where(region: "us")] }  # materialize hot partitions at deploy
+  end
+
+Provision the (empty) cache table with a migration generated from the
+relation, so the table exists at deploy time:
+
+  bin/rails generate activerecord_materialized:migration SalesSummary
+  bin/rails db:migrate
+
+Verify the cache tables still match their relations (e.g. in CI / on deploy):
+
+  bin/rails materialized:verify
+
+Build the view once (the only full-scan path — never happens implicitly; runs
+in the database via INSERT ... SELECT):
+
+  SalesSummary.rebuild!(confirm: true)
+
+Or warm just the hot partitions at deploy time (leaving the rest to read
+through on demand):
+
+  bin/rails materialized:warm_up
+
+Until a view is built, reads transparently fall through to the source query
+(cold_read :read_through). After it is built, writes to depends_on tables
+incrementally maintain it; refresh! applies that maintenance (it never rebuilds):
+
+  SalesSummary.refresh!            # incremental maintenance only
+  SalesSummary.refresh_if_stale!
+
+Or via Rake:
+
+  bin/rails materialized:rebuild       # intentional full materialization
+  bin/rails materialized:refresh_all   # incremental maintenance pass
+  bin/rails materialized:refresh_stale

data/lib/generators/activerecord_materialized/install/templates/create_ar_materialized_view_metadata.rb.erb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+class CreateArMaterializedViewMetadata < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :ar_materialized_view_metadata do |t|
+      t.string :view_name, null: false
+      t.datetime :last_refreshed_at
+      t.boolean :refreshing, null: false, default: false
+      t.boolean :dirty, null: false, default: true
+      t.boolean :warm, null: false, default: false
+      t.integer :row_count
+      t.integer :refresh_duration_ms
+      t.text :last_error
+      t.text :maintenance_payload
+
+      t.timestamps
+    end
+
+    add_index :ar_materialized_view_metadata, :view_name, unique: true
+
+    create_table :ar_materialized_view_partitions do |t|
+      t.string :view_name, null: false
+      t.string :partition_key, null: false
+
+      t.datetime :created_at, null: false
+    end
+
+    add_index :ar_materialized_view_partitions, %i[view_name partition_key], unique: true
+  end
+end

data/lib/generators/activerecord_materialized/install_generator.rb

@@ -0,0 +1,32 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/migration"
+
+module ActiverecordMaterialized
+  # `rails generate activerecord_materialized:install` — installs the metadata-table migration.
+  class InstallGenerator < ::Rails::Generators::Base
+    extend T::Sig
+    include Rails::Generators::Migration
+
+    source_root File.expand_path("install", __dir__)
+
+    sig { params(dirname: String).returns(String) }
+    def self.next_migration_number(dirname)
+      next_migration_number = T.unsafe(self).current_migration_number(dirname) + 1
+      ::ActiveRecord::Migration.next_migration_number(next_migration_number)
+    end
+
+    sig { void }
+    def copy_migration
+      migration_template "create_ar_materialized_view_metadata.rb.erb",
+                         "db/migrate/create_ar_materialized_view_metadata.rb"
+    end
+
+    sig { void }
+    def show_readme
+      readme "README" if behavior == :invoke
+    end
+  end
+end

data/lib/generators/activerecord_materialized/migration_generator.rb

@@ -0,0 +1,51 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/migration"
+
+module ActiverecordMaterialized
+  # Generates a migration that provisions a materialized view's (empty) cache
+  # table, with columns inferred from the view's source relation:
+  #
+  #   bin/rails generate activerecord_materialized:migration SalesSummary
+  #
+  # Run after the view's `materialized_from` is defined, then `db:migrate`.
+  class MigrationGenerator < ::Rails::Generators::NamedBase
+    extend T::Sig
+    include ::Rails::Generators::Migration
+
+    source_root File.expand_path("templates", __dir__)
+
+    sig { params(dirname: String).returns(String) }
+    def self.next_migration_number(dirname)
+      ::ActiveRecord::Migration.next_migration_number(T.unsafe(self).current_migration_number(dirname) + 1)
+    end
+
+    sig { void }
+    def create_migration_file
+      migration_template "materialized_view_migration.rb.erb",
+                         File.join("db", "migrate", "create_#{builder.table_name}.rb")
+    end
+
+    private
+
+    sig { returns(::ActiveRecord::Materialized::MigrationBuilder) }
+    def builder
+      @builder ||= T.let(build_builder, T.nilable(::ActiveRecord::Materialized::MigrationBuilder))
+    end
+
+    # Resolve via the registry (after eager-load) rather than constantize.
+    sig { returns(::ActiveRecord::Materialized::MigrationBuilder) }
+    def build_builder
+      T.unsafe(::Rails).application.eager_load!
+      view_class = ::ActiveRecord::Materialized::Registry.for_class_name(class_name)
+      if view_class.nil?
+        Kernel.raise ::Thor::Error,
+                     "Unknown materialized view: #{class_name}. Define it before generating its migration."
+      end
+
+      ::ActiveRecord::Materialized::MigrationBuilder.new(view_class)
+    end
+  end
+end

data/lib/generators/activerecord_materialized/templates/materialized_view.rb.erb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class <%= class_name %> < ActiveRecord::Materialized::View
+  self.table_name = "mv_<%= file_name %>"
+
+  # Replace ApplicationRecord.none with your ActiveRecord::Relation source, e.g.:
+  #   Order.group(:status).select(
+  #     Order.arel_table[:status],
+  #     ActiveRecord::Materialized::QueryExpressions.count_as(Order.arel_table[:id], as: :order_count)
+  #   )
+  materialized_from { ApplicationRecord.none }
+
+  depends_on :application_record
+  refresh_on_change :async
+  refresh_debounce 2.seconds
+  max_staleness 12.hours
+end

data/lib/generators/activerecord_materialized/templates/materialized_view_migration.rb.erb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class <%= builder.migration_class_name %> < ActiveRecord::Migration[<%= builder.migration_version %>]
+  def change
+    create_table :<%= builder.table_name %> do |t|
+<% builder.column_definitions.each do |column| -%>
+      t.<%= column.type %> :<%= column.name %>
+<% end -%>
+    end
+  end
+end

data/lib/generators/activerecord_materialized/view_generator.rb

@@ -0,0 +1,18 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module ActiverecordMaterialized
+  # `rails generate activerecord_materialized:view NAME` — scaffolds a materialized view class.
+  class ViewGenerator < ::Rails::Generators::NamedBase
+    extend T::Sig
+
+    source_root File.expand_path("templates", __dir__)
+
+    sig { void }
+    def create_model
+      template "materialized_view.rb.erb", File.join("app/models", class_path, "#{file_name}.rb")
+    end
+  end
+end

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification
+name: activerecord-materialized
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Michael Avrukin
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '8.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '8.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '8.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '8.0'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '8.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '8.0'
+- !ruby/object:Gem::Dependency
+  name: sorbet-runtime
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.5'
+description: |
+  Provides transparent materialized view semantics for Rails applications
+  on databases that do not support native materialized views. Precomputes
+  expensive analytical queries into cache tables with scheduled or on-demand refresh.
+email:
+- michael@avrukin.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- lib/activerecord/materialized.rb
+- lib/activerecord/materialized/aggregate_analysis.rb
+- lib/activerecord/materialized/async_refresher.rb
+- lib/activerecord/materialized/cache_table_schema.rb
+- lib/activerecord/materialized/cold_read.rb
+- lib/activerecord/materialized/configuration.rb
+- lib/activerecord/materialized/delta_maintainer.rb
+- lib/activerecord/materialized/dependency_registry.rb
+- lib/activerecord/materialized/dependency_trackable.rb
+- lib/activerecord/materialized/incremental_maintainer.rb
+- lib/activerecord/materialized/maintenance_delta.rb
+- lib/activerecord/materialized/maintenance_delta_builder.rb
+- lib/activerecord/materialized/maintenance_store.rb
+- lib/activerecord/materialized/metadata.rb
+- lib/activerecord/materialized/metadata/maintenance_payload.rb
+- lib/activerecord/materialized/metadata/schema.rb
+- lib/activerecord/materialized/metadata/timestamps.rb
+- lib/activerecord/materialized/metadata_record.rb
+- lib/activerecord/materialized/migration_builder.rb
+- lib/activerecord/materialized/module_api.rb
+- lib/activerecord/materialized/partition_record.rb
+- lib/activerecord/materialized/partition_state.rb
+- lib/activerecord/materialized/query_expressions.rb
+- lib/activerecord/materialized/railtie.rb
+- lib/activerecord/materialized/refresh_callbacks.rb
+- lib/activerecord/materialized/refresh_job.rb
+- lib/activerecord/materialized/refresh_result.rb
+- lib/activerecord/materialized/refresh_scheduler.rb
+- lib/activerecord/materialized/refresher.rb
+- lib/activerecord/materialized/registry.rb
+- lib/activerecord/materialized/relation_cache_writer.rb
+- lib/activerecord/materialized/schema_verifier.rb
+- lib/activerecord/materialized/summary_delta.rb
+- lib/activerecord/materialized/summary_delta_builder.rb
+- lib/activerecord/materialized/table_model_registry.rb
+- lib/activerecord/materialized/tasks.rb
+- lib/activerecord/materialized/type_reexports.rb
+- lib/activerecord/materialized/version.rb
+- lib/activerecord/materialized/view.rb
+- lib/activerecord/materialized/view_class.rb
+- lib/activerecord/materialized/view_configuration_class_methods.rb
+- lib/activerecord/materialized/view_definition.rb
+- lib/activerecord/materialized/view_incremental_class_methods.rb
+- lib/activerecord/materialized/view_query_access_class_methods.rb
+- lib/activerecord/materialized/view_refresh_policy_class_methods.rb
+- lib/activerecord/materialized/write_change.rb
+- lib/activerecord_materialized_types.rb
+- lib/generators/activerecord_materialized/install/templates/README
+- lib/generators/activerecord_materialized/install/templates/create_ar_materialized_view_metadata.rb.erb
+- lib/generators/activerecord_materialized/install_generator.rb
+- lib/generators/activerecord_materialized/migration_generator.rb
+- lib/generators/activerecord_materialized/templates/materialized_view.rb.erb
+- lib/generators/activerecord_materialized/templates/materialized_view_migration.rb.erb
+- lib/generators/activerecord_materialized/view_generator.rb
+homepage: https://github.com/mavrukin/activerecord-materialized
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  source_code_uri: https://github.com/mavrukin/activerecord-materialized
+  changelog_uri: https://github.com/mavrukin/activerecord-materialized/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/mavrukin/activerecord-materialized/issues
+  documentation_uri: https://rubydoc.info/gems/activerecord-materialized
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.15
+specification_version: 4
+summary: Application-level materialized views for ActiveRecord
+test_files: []