grain 0.0.1

data/lib/generators/grain/rollup/templates/rollup.rb.erb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class <%= rollup_class_name %> < Grain::Rollup
+  # The table the measures are read from and whose rows are counted. `where`
+  # narrows it, and may reach through a belongs_to.
+  #
+  #   fact LineItem, where: { order: { state: "paid" } }
+  fact :TODO
+
+  # The column the rollup is partitioned by. Required: starting the key with the
+  # most selective column is what keeps reads and recomputes cheap.
+  #
+  #   tenant :store_id, via: { order: :store_id }
+  tenant :TODO, via: :TODO
+
+  # Optional. Without it the rollup is a running total per dimension — a counter
+  # cache that can be verified instead of drifting.
+  #
+  #   time :ordered_on, via: { order: :placed_at }, grain: :day
+
+  # Dimensions are resolved by following belongs_to associations upward from the
+  # fact, up to three hops. `via` can also name a column on the fact itself.
+  # Mark one immutable to promise it never changes after the row is created, and
+  # Grain will skip watching that table.
+  #
+  #   dimension :product_id,  via: :product_id
+  #   dimension :category_id, via: { product: :category_id }
+  #   dimension :currency,    via: { order: :currency }, immutable: true
+
+  # count needs no type. Every other aggregate runs over SQL whose type cannot be
+  # inferred, and guessing would mean silently rounding your own numbers.
+  #
+  #   measure :line_count,    count: true
+  #   measure :revenue_cents, sum: "quantity * unit_price_cents", type: :bigint
+
+  # Ratios are stored as their two parts and divided on read, so a rate stays
+  # correct at every grain rather than being frozen at the one it was computed for.
+  #
+  #   ratio :average_unit_price, of: :revenue_cents, over: :units
+end

data/lib/generators/grain/table/table_generator.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Grain
+  module Generators
+    # Builds the migration for a rollup that already exists: its table, and the
+    # triggers on every table its definition depends on.
+    #
+    # Run it again whenever the definition changes. The rollup's shape is derived
+    # from the class, so the generated migration always matches what the code
+    # currently declares.
+    class TableGenerator < Rails::Generators::NamedBase
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Generates the migration that creates NAME's rollup table and its triggers."
+
+      def create_table_migration
+        migration_template "create_rollup_table.rb.erb", "db/migrate/#{rollup_migration_basename}.rb"
+      end
+
+      def report_trigger_tables
+        say ""
+        say "Triggers will be attached to:", :green
+        triggers.specs.each do |spec|
+          scope = spec.narrowed? ? "updates of #{spec.update_columns.join(", ")}" : "all updates"
+          say "  #{spec.table} (#{scope})"
+        end
+        say ""
+      end
+
+      private
+
+      def rollup
+        @rollup ||= resolve_rollup
+      end
+
+      # Thor prints the message and stops, which is the right behaviour for a
+      # command line; the rule itself lives in RollupLookup so it can be tested.
+      def resolve_rollup
+        Grain::RollupLookup.find!(name)
+      rescue Grain::RollupNotFoundError => e
+        raise Thor::Error, e.message
+      end
+
+      def definition
+        @definition ||= rollup.validate!
+      end
+
+      def migration
+        @migration ||= Grain::Migration.new(definition)
+      end
+
+      # Every rollup in the application, not just this one. The trigger on a
+      # table is shared, so narrowing it to what this rollup happens to need
+      # would silently break any other rollup that needs more.
+      def triggers
+        @triggers ||= Grain::Triggers.new(all_definitions)
+      end
+
+      def all_definitions
+        (Grain::Registry.all + [rollup]).uniq.map(&:definition)
+      end
+
+      # Deliberately not called migration_file_name or migration_class_name:
+      # Rails' own migration machinery calls methods by those names on the
+      # generator, and shadowing them breaks it from the inside.
+      def rollup_migration_basename
+        "create_#{migration.table_name}"
+      end
+
+      def rollup_migration_class_name
+        rollup_migration_basename.camelize
+      end
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+
+      def indented(text, spaces)
+        prefix = " " * spaces
+        text.each_line.map { |line| line.strip.empty? ? line : "#{prefix}#{line}" }.join
+      end
+
+      def table_body
+        indented(migration.up, 4).rstrip
+      end
+
+      def executed_statements(statements, spaces)
+        statements.map do |statement|
+          "#{" " * spaces}execute <<~SQL\n#{indented(statement, spaces + 2)}\n#{" " * spaces}SQL"
+        end.join("\n\n")
+      end
+
+      def trigger_up
+        executed_statements(triggers.up_statements, 4)
+      end
+
+      def trigger_down
+        executed_statements(triggers.down_statements, 4)
+      end
+
+      def drop_table_statement
+        migration.down
+      end
+    end
+  end
+end

data/lib/generators/grain/table/templates/create_rollup_table.rb.erb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Written by `rails generate grain:table <%= file_name %>` from
+# <%= rollup.name %>.
+#
+# Regenerate this whenever that definition changes: the table's shape and the
+# tables it watches are both derived from it.
+class <%= rollup_migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def up
+<%= table_body %>
+
+<%= trigger_up %>
+  end
+
+  def down
+<%= trigger_down %>
+
+    <%= drop_table_statement %>
+  end
+end

data/lib/grain/backfill.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Grain
+  # Populates a rollup from data that already exists.
+  #
+  # A new rollup starts empty: its triggers only see what happens next. The
+  # backfill is what makes it true about the past.
+  #
+  # The work is sliced rather than batched by row. Rows belonging to one cell are
+  # scattered through the fact table, so a batch of rows would have to add to
+  # cells already written, which is the delta problem again with none of its
+  # safeguards. A slice — one day, or one tenant — is instead rebuilt whole with
+  # the same recompute the worker uses: idempotent, never leaving a cell showing a
+  # partial total, and safe to run while the worker is running.
+  #
+  # Resuming after an interruption is manual and deliberate: slices are processed
+  # in order, so passing `from:` the last one reported picks up where it stopped.
+  # Re-running an already-done slice is harmless either way.
+  class Backfill
+    attr_reader :rollup, :definition
+
+    def initialize(rollup, from: nil, pause: 0)
+      @rollup = rollup
+      @definition = rollup.definition.validate!
+      @projection = Projection.new(@definition)
+      @from = from
+      @pause = pause
+    end
+
+    # Yields each slice value as it completes, so a caller can report progress or
+    # record where to resume from. Returns the number of slices rebuilt.
+    def call
+      recompute = Recompute.new(definition)
+      slices.each_with_index do |value, index|
+        recompute.call_slice(slice_dimension, value)
+        yield(value, index + 1, slices.length) if block_given?
+        sleep(@pause) if @pause.positive?
+      end
+      slices.length
+    end
+
+    # The dimension the work is cut along: the time bucket when there is one,
+    # since a day is a naturally bounded unit of work, and the tenant otherwise.
+    def slice_dimension
+      definition.temporal? ? definition.time : definition.tenant
+    end
+
+    # Distinct slice values that actually have data, in order.
+    #
+    # This is the expensive part of a backfill: it reads the fact table to find
+    # them. It is one pass and far cheaper than the full aggregate, and using the
+    # distinct values rather than a min-to-max range skips every gap.
+    def slices
+      @slices ||= load_slices
+    end
+
+    private
+
+    def load_slices
+      expression = @projection.dimension_expression(slice_dimension)
+      sql = +"SELECT DISTINCT #{expression} AS slice FROM #{@projection.from_and_joins.join(" ")}"
+      conditions = @projection.filter_conditions
+      conditions << "#{expression} >= #{quote(@from)}" unless @from.nil?
+      sql << " WHERE #{conditions.join(" AND ")}" if conditions.any?
+      sql << " ORDER BY slice"
+      connection.select_values(sql)
+    end
+
+    def quote(value)
+      connection.quote(value)
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end

data/lib/grain/cells.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Grain
+  # Finds the cells a change could have touched.
+  #
+  # The rule this is built on: recomputing a cell that did not need it is
+  # harmless, while missing one that did is the only unforgivable bug. So the
+  # search is deliberately generous — the fact's filter is applied when looking
+  # for where rows are now, and left off when looking for where they used to be.
+  class Cells
+    attr_reader :projection
+
+    def initialize(definition)
+      @definition = definition
+      @projection = Projection.new(definition)
+    end
+
+    # Cells the given fact rows belong to as things stand.
+    def live_for_facts(row_ids)
+      return [] if row_ids.empty?
+
+      select(conditions: [id_in(Projection::FACT, row_ids)] + projection.filter_conditions)
+    end
+
+    # The cell a fact row sat in before it changed, rebuilt from the log. Without
+    # this the row's contribution would sit in its old cell forever, because
+    # nothing points there any more.
+    def previous_for_fact(previous_json)
+      select(substitutions: { [] => previous_json })
+    end
+
+    # Cells reachable through a row of a watched table, as things stand.
+    def live_through(hops, row_ids)
+      return [] if row_ids.empty?
+
+      select(conditions: [id_in(projection.alias_for(hops), row_ids)] + projection.filter_conditions)
+    end
+
+    # ...and as they were, with that table's row rebuilt from the log.
+    def previous_through(hops, previous_json)
+      select(substitutions: { hops => previous_json })
+    end
+
+    private
+
+    def select(conditions: [], substitutions: {})
+      sql = +"SELECT DISTINCT #{selection} FROM #{projection.from_and_joins(substitutions).join(" ")}"
+      sql << " WHERE #{conditions.join(" AND ")}" if conditions.any?
+      connection.select_all(sql).to_a.map(&:symbolize_keys)
+    end
+
+    def selection
+      projection.key_columns.zip(projection.dimension_expressions)
+                .map { |name, expression| "#{expression} AS #{name}" }
+                .join(", ")
+    end
+
+    def id_in(table_alias, row_ids)
+      "#{table_alias}.id IN (#{row_ids.map { |id| connection.quote(id) }.join(", ")})"
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end

data/lib/grain/change_log.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Grain
+  # The single table every trigger writes into, the trigger function they all
+  # share, and the vocabulary of what a trigger can record.
+  #
+  # One trigger per source table, never one per rollup: several rollups can read
+  # the same fact table, and triggers must not multiply with them. A trigger
+  # records that a row changed and nothing more. Deciding which rollups care, and
+  # which of their cells are affected, is the worker's job.
+  module ChangeLog
+    OPERATIONS = %i[insert update delete].freeze
+
+    FUNCTION_NAME = "grain_record_change"
+
+    COLUMNS = {
+      id: :bigserial,
+      source_table: :text,
+      # Text rather than bigint so that Grain works on integer, UUID and string
+      # primary keys alike. The worker casts it back when it reads the source, so
+      # the fact table's own index is still used.
+      row_id: :text,
+      operation: :text,
+      # The row as it was before the change, recorded for updates and deletes.
+      #
+      # Without it, the cell a row is leaving cannot be located: once the row
+      # carries its new values, nothing points back at where it used to be
+      # counted, and that cell would keep the departed row in its totals
+      # forever. This is why the change log is not merely a list of ids.
+      previous: :jsonb,
+      created_at: :timestamptz
+    }.freeze
+
+    class << self
+      def table_name
+        Grain.config.change_log_table
+      end
+
+      def operation!(name)
+        operation = name.to_s.downcase.to_sym
+        return operation if OPERATIONS.include?(operation)
+
+        raise Error, "unknown change log operation #{name.inspect}, expected #{OPERATIONS.inspect}"
+      end
+
+      # Updates and deletes need the previous row to locate the cell being left.
+      # An insert leaves nothing behind, so it needs none.
+      def previous_required?(operation)
+        operation!(operation) != :insert
+      end
+
+      # No secondary indexes: the worker reads forward by id and prunes by id, so
+      # the primary key already covers both. The log is drained continuously and
+      # is never meant to grow.
+      def table_definition
+        <<~RUBY
+          create_table :#{table_name} do |t|
+            t.text :source_table, null: false
+            t.text :row_id, null: false
+            t.text :operation, null: false
+            t.jsonb :previous
+            t.datetime :created_at, null: false
+          end
+        RUBY
+      end
+
+      # One function for every watched table. TG_TABLE_NAME tells the worker which
+      # table a row came from, so nothing here has to be generated per table.
+      #
+      # clock_timestamp() rather than now(): now() returns the transaction's start
+      # time, which would stamp every row written in one transaction identically.
+      def function_sql
+        <<~SQL
+          CREATE OR REPLACE FUNCTION #{FUNCTION_NAME}() RETURNS trigger
+          LANGUAGE plpgsql AS $$
+          BEGIN
+            IF TG_OP = 'INSERT' THEN
+              INSERT INTO #{table_name} (source_table, row_id, operation, previous, created_at)
+              VALUES (TG_TABLE_NAME, NEW.id::text, 'insert', NULL, clock_timestamp());
+              RETURN NEW;
+            ELSIF TG_OP = 'UPDATE' THEN
+              INSERT INTO #{table_name} (source_table, row_id, operation, previous, created_at)
+              VALUES (TG_TABLE_NAME, NEW.id::text, 'update', to_jsonb(OLD), clock_timestamp());
+              RETURN NEW;
+            ELSE
+              INSERT INTO #{table_name} (source_table, row_id, operation, previous, created_at)
+              VALUES (TG_TABLE_NAME, OLD.id::text, 'delete', to_jsonb(OLD), clock_timestamp());
+              RETURN OLD;
+            END IF;
+          END;
+          $$;
+        SQL
+      end
+
+      def drop_function_sql
+        "DROP FUNCTION IF EXISTS #{FUNCTION_NAME}();"
+      end
+    end
+  end
+end

data/lib/grain/configuration.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Grain
+  # Application-wide settings. Defaults are chosen to be safe on a busy
+  # production database rather than fast on an idle one.
+  class Configuration
+    # Table that database triggers write change events into.
+    attr_accessor :change_log_table
+
+    # Rows per batch when backfilling or draining the change log. Small enough
+    # to keep transactions short and avoid long lock holds.
+    attr_accessor :batch_size
+
+    # Seconds a delta worker may run before yielding, so a large backlog cannot
+    # monopolise a job queue slot.
+    attr_accessor :max_run_seconds
+
+    # Time zone the day buckets are cut in. A timestamp has to be resolved to a
+    # calendar day in some zone, and leaving it to the database session would make
+    # the same row land in different buckets depending on who ran the query.
+    attr_accessor :time_zone
+
+    # Where Grain writes its own diagnostics.
+    attr_accessor :logger
+
+    def initialize
+      @change_log_table = "grain_change_log"
+      @batch_size = 1_000
+      @max_run_seconds = 30
+      @time_zone = "UTC"
+      @logger = nil
+    end
+  end
+end

data/lib/grain/definition.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Grain
+  # Everything a Rollup subclass declares, plus what can be derived from it: the
+  # rollup table's name and key order, and which tables have to be watched to
+  # keep the aggregate correct.
+  class Definition
+    TABLE_PREFIX = "grain_"
+
+    attr_reader :owner, :fact, :dimensions, :measures, :ratios
+
+    def initialize(owner)
+      @owner = owner
+      @fact = nil
+      @dimensions = []
+      @measures = []
+      @ratios = []
+    end
+
+    def declare_fact(model, where: nil)
+      raise InvalidDefinitionError, "#{owner} already declares a fact" if fact
+
+      @fact = Fact.new(model, where: where)
+    end
+
+    def add_dimension(name, via:, role: :dimension, grain: nil, immutable: false)
+      dimension = Dimension.new(name: name, via: via, role: role, grain: grain, immutable: immutable)
+      reject_duplicate_name!(dimension.name)
+      reject_duplicate_role!(dimension)
+      @dimensions << dimension
+      dimension
+    end
+
+    def add_measure(name, options)
+      measure = Measure.from_options(name, options)
+      reject_duplicate_name!(measure.name)
+      @measures << measure
+      measure
+    end
+
+    def add_ratio(name, numerator:, denominator:)
+      ratio = Ratio.new(name: name, numerator: numerator, denominator: denominator)
+      reject_duplicate_name!(ratio.name)
+      @ratios << ratio
+      ratio
+    end
+
+    def tenant
+      dimensions.find(&:tenant?)
+    end
+
+    def time
+      dimensions.find(&:time?)
+    end
+
+    # False for a rollup that buckets by nothing but its dimensions — a counter
+    # cache. Reads on one cannot take a date range, and it has no late-arriving
+    # data to worry about.
+    def temporal?
+      !time.nil?
+    end
+
+    def plain_dimensions
+      dimensions.reject { |dimension| dimension.tenant? || dimension.time? }
+    end
+
+    # The rollup table's primary key, in a fixed order so that migrations,
+    # delta updates and reads all agree: tenant, then time bucket, then the
+    # plain dimensions in declaration order.
+    def key_dimensions
+      [tenant, time, *plain_dimensions].compact
+    end
+
+    def table_name
+      raise InvalidDefinitionError, "a rollup needs a name to derive its table from" if anonymous?
+
+      "#{TABLE_PREFIX}#{owner.name.underscore.pluralize}"
+    end
+
+    # Paths whose tables can move fact rows between cells.
+    def watched_paths
+      dimensions.select(&:watched?).map(&:path).uniq
+    end
+
+    # Every association hop along a watched path, not only the first.
+    #
+    # A change anywhere along the path moves fact rows between cells: if a
+    # dimension resolves as order then store then currency, changing that store's
+    # currency moves every line item on every order pointing at it. Watching only
+    # the root would let that drift silently, which is the worst failure this
+    # gem can have.
+    def watched_associations
+      watched_paths.flat_map(&:hops).uniq
+    end
+
+    # Associations the fact's filter reaches through. A change there adds or
+    # removes fact rows entirely, which no delta can express.
+    def filter_associations
+      return [] unless fact
+
+      fact.filter_associations
+    end
+
+    # True when a change somewhere other than the fact table can add or remove
+    # fact rows entirely, which no delta can express.
+    def invalidated_by_filter?
+      fact ? fact.filtered_through_association? : false
+    end
+
+    def measure_names
+      measures.map(&:name)
+    end
+
+    def validate!
+      DefinitionValidator.new(self).validate!
+    end
+
+    private
+
+    def anonymous?
+      owner.name.nil? || owner.name.empty?
+    end
+
+    def declared_names
+      dimensions.map(&:name) + measure_names + ratios.map(&:name)
+    end
+
+    def reject_duplicate_name!(name)
+      return unless declared_names.include?(name)
+
+      raise InvalidDefinitionError, "#{owner} already declares #{name.inspect}"
+    end
+
+    def reject_duplicate_role!(dimension)
+      return unless (dimension.tenant? && tenant) || (dimension.time? && time)
+
+      raise InvalidDefinitionError, "#{owner} already declares a #{dimension.role}"
+    end
+  end
+end

data/lib/grain/definition_validator.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Grain
+  # Judges whether a definition is complete and internally consistent.
+  #
+  # Kept apart from Definition so that accumulating declarations and deciding
+  # whether they add up stay separate concerns: the DSL builds, this rules.
+  class DefinitionValidator
+    # Parts a rollup cannot do without.
+    #
+    # A time dimension is deliberately absent: a rollup without one is a counter
+    # cache that cannot drift, which is a use case in its own right. A tenant is
+    # required, because starting the key with the most selective column is what
+    # makes reads and scoped recomputes cheap.
+    REQUIRED_PARTS = {
+      fact: "declares no fact",
+      tenant: "declares no tenant"
+    }.freeze
+
+    attr_reader :definition
+
+    def initialize(definition)
+      @definition = definition
+    end
+
+    def validate!
+      validate_required_parts!
+      validate_ratios!
+      definition
+    end
+
+    private
+
+    def owner
+      definition.owner
+    end
+
+    def validate_required_parts!
+      REQUIRED_PARTS.each do |reader, complaint|
+        raise InvalidDefinitionError, "#{owner} #{complaint}" if definition.public_send(reader).nil?
+      end
+
+      raise InvalidDefinitionError, "#{owner} declares no measures" if definition.measures.empty?
+    end
+
+    def validate_ratios!
+      definition.ratios.each { |ratio| validate_ratio!(ratio) }
+    end
+
+    def validate_ratio!(ratio)
+      [ratio.numerator, ratio.denominator].each do |part|
+        next if definition.measure_names.include?(part)
+
+        raise InvalidDefinitionError, "ratio #{ratio.name} refers to unknown measure #{part.inspect}"
+      end
+    end
+  end
+end