grain 0.0.1

data/lib/grain/triggers.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Grain
+  # Works out which tables need a trigger, which columns of each are worth
+  # reacting to, and renders the SQL that attaches them.
+  #
+  # Built from every rollup that touches a table, not from one at a time. The
+  # trigger name is per table because the function it calls is shared, so a
+  # migration that considered only its own rollup would narrow a trigger another
+  # rollup depends on and leave that one drifting in silence. The column list is
+  # therefore the union across all of them.
+  class Triggers
+    # `update_columns` narrows the UPDATE trigger to the columns that can move a
+    # row between cells; nil means every update has to be logged.
+    Spec = Struct.new(:table, :update_columns, keyword_init: true) do
+      def trigger_name
+        "grain_#{table}_changed"
+      end
+
+      def narrowed?
+        !update_columns.nil?
+      end
+    end
+
+    attr_reader :definitions
+
+    def initialize(definitions)
+      @definitions = Array(definitions).map(&:validate!).uniq
+    end
+
+    # Fact tables first, then the tables reached through them.
+    def specs
+      fact_specs + related_specs
+    end
+
+    # One statement per entry, so a migration can execute them individually
+    # instead of relying on the adapter accepting several at once.
+    def up_statements
+      specs.flat_map { |spec| [drop_sql(spec), create_sql(spec)] }
+    end
+
+    def down_statements
+      specs.map { |spec| drop_sql(spec) }
+    end
+
+    def up
+      up_statements.join("\n")
+    end
+
+    def down
+      down_statements.join("\n")
+    end
+
+    private
+
+    # Measures aggregate arbitrary SQL, so which of a fact table's columns feed
+    # them cannot be known. Narrowing here risks missing an update and letting a
+    # rollup drift, so every update on a fact table is logged. Being a fact for
+    # any one rollup is enough to disqualify the table from narrowing.
+    def fact_specs
+      fact_tables.map { |table| Spec.new(table: table, update_columns: nil) }
+    end
+
+    def fact_tables
+      definitions.map { |definition| TypeResolver.new(definition).fact_table }.uniq
+    end
+
+    def related_specs
+      union_of_related_columns.reject { |table, _| fact_tables.include?(table) }
+                              .map { |table, columns| Spec.new(table: table, update_columns: columns.sort) }
+    end
+
+    def union_of_related_columns
+      definitions.each_with_object({}) do |definition, union|
+        WatchedColumns.new(definition).to_h.each do |table, columns|
+          union[table] = ((union[table] || []) + columns).uniq
+        end
+      end
+    end
+
+    def create_sql(spec)
+      <<~SQL.strip
+        CREATE TRIGGER #{spec.trigger_name}
+        AFTER INSERT OR #{update_clause(spec)} OR DELETE ON #{spec.table}
+        FOR EACH ROW EXECUTE FUNCTION #{ChangeLog::FUNCTION_NAME}();
+      SQL
+    end
+
+    def update_clause(spec)
+      return "UPDATE" unless spec.narrowed?
+
+      "UPDATE OF #{spec.update_columns.join(", ")}"
+    end
+
+    def drop_sql(spec)
+      "DROP TRIGGER IF EXISTS #{spec.trigger_name} ON #{spec.table};"
+    end
+  end
+end

data/lib/grain/type_resolver.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Grain
+  # Works out the column type of every column of a rollup table by walking each
+  # dimension's declared path to the model that owns the source column.
+  #
+  # This is the one part of the schema layer that needs the application's models,
+  # which is why it is separate from Schema: the shape of a rollup can be reasoned
+  # about without a database, its types cannot.
+  class TypeResolver
+    # A time dimension does not store the source timestamp, it stores the bucket
+    # the row falls into, so its type comes from the grain rather than the source.
+    BUCKET_TYPES = { day: :date }.freeze
+
+    attr_reader :definition
+
+    def initialize(definition)
+      @definition = definition
+    end
+
+    def dimension_type(dimension)
+      return BUCKET_TYPES.fetch(dimension.grain) if dimension.time?
+
+      source_column(dimension).type
+    end
+
+    # The type of the column a dimension is read from, before any bucketing. A
+    # time dimension needs this to know whether the source is already a calendar
+    # day or a timestamp that has to be resolved to one.
+    def source_type(dimension)
+      source_column(dimension).type
+    end
+
+    # Whether the source column can be null, which decides whether the rollup can
+    # use a plain composite primary key: Postgres will not accept a null in one.
+    def dimension_nullable?(dimension)
+      return false if dimension.time?
+
+      source_column(dimension).null
+    end
+
+    def measure_type(measure)
+      measure.type
+    end
+
+    def fact_table
+      definition.fact.model.table_name
+    end
+
+    def nullable_dimensions
+      definition.key_dimensions.select { |dimension| dimension_nullable?(dimension) }
+    end
+
+    # The validated belongs_to reflection for one hop. Public because working out
+    # which columns to watch needs the same walk and the same complaints.
+    def reflection!(model, hop, context)
+      reflection = model.reflect_on_association(hop)
+      raise InvalidDefinitionError, "#{model} has no association #{hop.inspect} (via #{context})" if reflection.nil?
+      raise InvalidDefinitionError, one_value_per_dimension(model, hop, context) unless reflection.belongs_to?
+
+      reflection
+    end
+
+    private
+
+    def one_value_per_dimension(model, hop, context)
+      "#{model}##{hop} is not a belongs_to (via #{context}). A fact row must resolve to one " \
+        "value per dimension, or it would be counted in several cells at once."
+    end
+
+    def source_column(dimension)
+      model = walk(definition.fact.model, dimension.path)
+      model.columns_hash.fetch(dimension.path.column.to_s) do
+        raise InvalidDefinitionError,
+              "#{dimension.name} resolves to #{model}##{dimension.path.column}, which does not exist"
+      end
+    end
+
+    def walk(model, path)
+      path.hops.reduce(model) { |current, hop| reflection!(current, hop, path).klass }
+    end
+  end
+end

data/lib/grain/verification.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Grain
+  # Recomputes a rollup from its source and reports every cell that disagrees.
+  #
+  # This is the point of the whole gem, not a diagnostic bolted on afterwards.
+  # Nobody puts an aggregation layer in front of numbers that matter without a way
+  # to prove it still tells the truth, so the obstacle Grain has to clear is never
+  # speed — it is doubt.
+  #
+  # A full verification is an aggregate scan of the source, which is a maintenance
+  # operation rather than something to run per request. Scope it by tenant or by
+  # date range on a large rollup.
+  class Verification
+    attr_reader :rollup, :definition
+
+    def initialize(rollup, tenant: nil, between: nil)
+      @rollup = rollup
+      @definition = rollup.definition.validate!
+      @scope = { tenant: tenant, between: between }
+      validate_scope!
+    end
+
+    def call(repair: false)
+      report = VerificationReport.new(rollup: rollup, discrepancies: discrepancies)
+      return report unless repair && !report.clean?
+
+      report.with_repaired(Recompute.new(definition).call(report.discrepancies.map(&:cell)))
+    end
+
+    def discrepancies
+      connection.select_all(query.to_s).to_a.map { |row| build_discrepancy(row.symbolize_keys) }
+    end
+
+    def query
+      VerificationQuery.new(definition, **@scope)
+    end
+
+    private
+
+    def key_columns
+      definition.key_dimensions.map(&:name)
+    end
+
+    def measure_columns
+      definition.measures.map(&:name)
+    end
+
+    def build_discrepancy(row)
+      Discrepancy.new(
+        cell: key_columns.to_h { |key| [key, row[key]] },
+        stored: side(row, "stored"),
+        expected: side(row, "expected")
+      )
+    end
+
+    def side(row, name)
+      return nil unless row[:"in_#{name}"].to_i.positive?
+
+      measure_columns.to_h { |measure| [measure, row[:"#{name}_#{measure}"]] }
+    end
+
+    def validate_scope!
+      return if @scope[:between].nil? || definition.temporal?
+
+      raise Error, "#{rollup} has no time dimension, so it cannot be verified over a date range"
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end

data/lib/grain/verification_query.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Grain
+  # Builds the single statement that compares a rollup against its source.
+  #
+  # Stored and expected are stacked with UNION ALL and folded with GROUP BY rather
+  # than joined. A FULL OUTER JOIN would be the obvious shape, but Postgres only
+  # supports one over merge-joinable or hash-joinable conditions, and matching
+  # cells needs IS NOT DISTINCT FROM: a dimension resolved from a nullable column
+  # has null as a legitimate coordinate, and null = null is never true. Grouping
+  # sidesteps the limitation and gets the semantics for free, since GROUP BY
+  # already treats nulls as equal.
+  class VerificationQuery
+    attr_reader :definition, :projection
+
+    def initialize(definition, tenant: nil, between: nil)
+      @definition = definition
+      @projection = Projection.new(definition)
+      @tenant = tenant
+      @between = between
+    end
+
+    def to_s
+      <<~SQL
+        WITH combined AS (
+          #{stored_side}
+          UNION ALL
+          #{expected_side}
+        ), folded AS (
+          SELECT #{folded_selection}
+          FROM combined
+          GROUP BY #{key_positions}
+        )
+        SELECT * FROM folded
+        WHERE #{disagreement}
+        ORDER BY #{key_positions}
+      SQL
+    end
+
+    private
+
+    def key_columns
+      projection.key_columns
+    end
+
+    def measure_columns
+      projection.measure_columns
+    end
+
+    def key_positions
+      (1..key_columns.length).to_a.join(", ")
+    end
+
+    def stored_side
+      columns = (key_columns + measure_columns).map { |column| quote_column(column) }.join(", ")
+      conditions = scope_conditions { |column| quote_column(column) }
+      where = conditions.empty? ? "" : " WHERE #{conditions.join(" AND ")}"
+      "SELECT #{columns}, 1 AS grain_side FROM #{definition.table_name}#{where}"
+    end
+
+    def expected_side
+      <<~SQL.rstrip
+        SELECT #{expected_selection}, 2 AS grain_side
+          FROM #{projection.from_and_joins.join(" ")}#{expected_where}
+          GROUP BY #{projection.dimension_expressions.join(", ")}
+      SQL
+    end
+
+    def expected_selection
+      pairs = key_columns.zip(projection.dimension_expressions) +
+              measure_columns.zip(projection.aggregate_expressions)
+      pairs.map { |name, expression| "#{expression} AS #{quote_column(name)}" }.join(", ")
+    end
+
+    def expected_where
+      conditions = projection.filter_conditions + scope_conditions { |column| dimension_expression(column) }
+      conditions.empty? ? "" : "\n  WHERE #{conditions.join(" AND ")}"
+    end
+
+    # At most one row per side per cell, so MAX picks that row's value and the
+    # counts say whether the side contributed a row at all — which is what
+    # distinguishes "this side had nothing" from "this side had a null".
+    def folded_selection
+      (key_columns.map { |key| quote_column(key) } + measure_pickers + presence_counts).join(", ")
+    end
+
+    def measure_pickers
+      sides.flat_map do |side, index|
+        measure_columns.map do |measure|
+          "MAX(CASE WHEN grain_side = #{index} THEN #{quote_column(measure)} END) AS #{side}_#{measure}"
+        end
+      end
+    end
+
+    def presence_counts
+      sides.map { |side, index| "COUNT(*) FILTER (WHERE grain_side = #{index}) AS in_#{side}" }
+    end
+
+    def sides
+      { "stored" => 1, "expected" => 2 }
+    end
+
+    # A cell survives if one side never produced a row, or any measure differs.
+    def disagreement
+      absent = ["in_stored = 0", "in_expected = 0"]
+      differing = measure_columns.map { |measure| "stored_#{measure} IS DISTINCT FROM expected_#{measure}" }
+      (absent + differing).join(" OR ")
+    end
+
+    def scope_conditions
+      conditions = []
+      conditions << "#{yield(definition.tenant.name)} = #{quote(@tenant)}" unless @tenant.nil?
+      conditions << between_condition(yield(definition.time.name)) unless @between.nil?
+      conditions
+    end
+
+    def between_condition(column)
+      "#{column} BETWEEN #{quote(@between.first)} AND #{quote(@between.last)}"
+    end
+
+    def dimension_expression(name)
+      @dimension_expressions ||= key_columns.zip(projection.dimension_expressions).to_h
+      @dimension_expressions.fetch(name)
+    end
+
+    def quote_column(name)
+      ActiveRecord::Base.connection.quote_column_name(name)
+    end
+
+    def quote(value)
+      ActiveRecord::Base.connection.quote(value)
+    end
+  end
+end

data/lib/grain/verification_report.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Grain
+  # What a verification found, in a shape that suits both a person reading a
+  # terminal and a build deciding whether to fail.
+  class VerificationReport
+    attr_reader :rollup, :discrepancies, :repaired
+
+    def initialize(rollup:, discrepancies:, repaired: 0)
+      @rollup = rollup
+      @discrepancies = discrepancies.freeze
+      @repaired = repaired
+    end
+
+    def clean?
+      discrepancies.empty?
+    end
+
+    def count
+      discrepancies.length
+    end
+
+    def count_of(kind)
+      discrepancies.count { |discrepancy| discrepancy.kind == kind }
+    end
+
+    def with_repaired(number)
+      self.class.new(rollup: rollup, discrepancies: discrepancies, repaired: number)
+    end
+
+    def to_s
+      return "#{rollup}: agrees with its source" if clean?
+
+      [summary, *discrepancies.map { |discrepancy| "  #{discrepancy}" }].join("\n")
+    end
+
+    private
+
+    def summary
+      tally = Discrepancy::KINDS.filter_map do |kind|
+        found = count_of(kind)
+        "#{found} #{kind}" if found.positive?
+      end
+      repair_note = repaired.positive? ? ", #{repaired} repaired" : ""
+      "#{rollup}: #{count} cells disagree (#{tally.join(", ")})#{repair_note}"
+    end
+  end
+end

data/lib/grain/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Grain
+  VERSION = "0.0.1"
+end

data/lib/grain/watched_columns.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Grain
+  # Which columns of which tables can move a fact row between cells, and so are
+  # worth waking a trigger for.
+  #
+  # Unlike the fact table these can be narrowed exactly: at each hop along a
+  # dimension's path the only column that matters is the foreign key leading to
+  # the next hop, or the dimension's own column at the end. Narrowing is not an
+  # optimisation to add later — without it a single busy table would write a log
+  # row on every update it ever takes.
+  class WatchedColumns
+    attr_reader :definition, :resolver
+
+    def initialize(definition)
+      @definition = definition
+      @resolver = TypeResolver.new(definition)
+    end
+
+    # { table name => [column names] }, with the fact table left out: its measures
+    # aggregate arbitrary SQL, so which of its columns feed them cannot be known
+    # and every update on it has to be logged.
+    def to_h
+      collected = {}
+      definition.watched_paths.each { |path| add_path(path, collected) }
+      definition.filter_associations.each { |association| add_filter(association, collected) }
+      collected.delete(resolver.fact_table)
+      collected
+    end
+
+    private
+
+    def add_path(path, into)
+      path.hops.each_with_index.reduce(definition.fact.model) do |model, (hop, index)|
+        add_hop(model, hop, path, index, into)
+      end
+    end
+
+    # Two columns matter at each step: the foreign key on the table being left,
+    # and — once the walk reaches the end — the dimension's own column.
+    def add_hop(model, hop, path, index, into)
+      reflection = resolver.reflection!(model, hop, path)
+      add(into, model.table_name, reflection.foreign_key)
+      reflection.klass.tap do |next_model|
+        add(into, next_model.table_name, path.column) if last_hop?(path, index)
+      end
+    end
+
+    def last_hop?(path, index)
+      index == path.hops.length - 1
+    end
+
+    def add_filter(association, into)
+      reflection = resolver.reflection!(definition.fact.model, association, association)
+      add(into, definition.fact.model.table_name, reflection.foreign_key)
+      conditions(association).each_key { |column| add(into, reflection.klass.table_name, column) }
+    end
+
+    def conditions(association)
+      where = definition.fact.where
+      where[association] || where[association.to_s] || {}
+    end
+
+    def add(into, table, column)
+      list = (into[table] ||= [])
+      list << column.to_s
+      list.uniq!
+    end
+  end
+end

data/lib/grain/worker.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Grain
+  # Drains the change log and brings the affected cells back in line with the
+  # source.
+  #
+  # Claiming and applying happen in one transaction: the log rows are deleted and
+  # the rollups are rewritten together, so a crash rolls the deletions back and
+  # the work is simply done again. Claiming uses SKIP LOCKED, so several workers
+  # can drain the same log without waiting on each other or doing the same work
+  # twice.
+  class Worker
+    Entry = Struct.new(:source_table, :row_id, :operation, :previous, keyword_init: true)
+
+    class << self
+      # Drains until the log is empty or the time budget runs out. Returns the
+      # number of log entries applied.
+      def drain(limit: Grain.config.batch_size, max_seconds: Grain.config.max_run_seconds)
+        applied = 0
+        deadline = monotonic + max_seconds
+        loop do
+          batch = new(limit: limit).call
+          applied += batch
+          break if batch.zero? || monotonic >= deadline
+        end
+        applied
+      end
+
+      def monotonic
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+
+    attr_reader :limit
+
+    def initialize(limit: Grain.config.batch_size)
+      @limit = limit
+    end
+
+    def call
+      connection.transaction do
+        entries = claim
+        next 0 if entries.empty?
+
+        apply(entries)
+        entries.length
+      end
+    end
+
+    private
+
+    # Deleting with RETURNING is the claim: inside the transaction the rows are
+    # gone, so no other worker can pick them up, and if this transaction fails
+    # they come back.
+    def claim
+      connection.select_all(claim_sql).to_a.map { |row| Entry.new(**row.symbolize_keys) }
+    end
+
+    def claim_sql
+      <<~SQL
+        DELETE FROM #{ChangeLog.table_name}
+        WHERE id IN (
+          SELECT id FROM #{ChangeLog.table_name}
+          ORDER BY id
+          LIMIT #{limit.to_i}
+          FOR UPDATE SKIP LOCKED
+        )
+        RETURNING source_table, row_id, operation, previous
+      SQL
+    end
+
+    def apply(entries)
+      entries.group_by(&:source_table).each do |table, group|
+        Registry.for_table(table).each { |rollup| refresh(rollup, table, group) }
+      end
+    end
+
+    def refresh(rollup, table, entries)
+      cells = affected_cells(rollup, table, entries)
+      Recompute.new(rollup.definition).call(cells)
+    end
+
+    def affected_cells(rollup, table, entries)
+      cells = Cells.new(rollup.definition)
+      as_fact(cells, table, entries) + as_watched(cells, table, entries)
+    end
+
+    def as_fact(cells, table, entries)
+      return [] unless cells.projection.fact_table == table
+
+      cells.live_for_facts(entries.map(&:row_id)) + previous_cells(entries) { |json| cells.previous_for_fact(json) }
+    end
+
+    def as_watched(cells, table, entries)
+      cells.projection.hops_for_table(table).flat_map do |hops|
+        cells.live_through(hops, entries.map(&:row_id)) +
+          previous_cells(entries) { |json| cells.previous_through(hops, json) }
+      end
+    end
+
+    def previous_cells(entries, &block)
+      entries.filter_map(&:previous).flat_map(&block)
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end

data/lib/grain.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/string/inflections"
+
+require_relative "grain/version"
+require_relative "grain/errors"
+require_relative "grain/configuration"
+require_relative "grain/path"
+require_relative "grain/dimension"
+require_relative "grain/measure"
+require_relative "grain/ratio"
+require_relative "grain/fact"
+require_relative "grain/definition_validator"
+require_relative "grain/definition"
+require_relative "grain/rollup"
+require_relative "grain/rollup_lookup"
+require_relative "grain/schema"
+require_relative "grain/change_log"
+require_relative "grain/type_resolver"
+require_relative "grain/migration"
+require_relative "grain/watched_columns"
+require_relative "grain/triggers"
+require_relative "grain/registry"
+require_relative "grain/join_graph"
+require_relative "grain/projection"
+require_relative "grain/cells"
+require_relative "grain/recompute"
+require_relative "grain/query_sql"
+require_relative "grain/query"
+require_relative "grain/backfill"
+require_relative "grain/worker"
+require_relative "grain/discrepancy"
+require_relative "grain/verification_report"
+require_relative "grain/verification_query"
+require_relative "grain/verification"
+
+# Grain keeps dashboard aggregates pre-computed and incrementally up to date
+# inside the application's own Postgres database.
+#
+# What exists so far is the definition layer: a rollup declares its fact, its
+# key dimensions and its measures, and Grain derives the rollup table's shape
+# and the set of tables that have to be watched. Nothing writes to a database
+# yet. See README.md for the design.
+module Grain
+  class << self
+    def config
+      @config ||= Configuration.new
+    end
+
+    def configure
+      yield config
+    end
+
+    def reset_config!
+      @config = Configuration.new
+    end
+  end
+end
+
+require_relative "grain/railtie" if defined?(Rails::Railtie)