grain 0.0.1

data/lib/grain/dimension.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Grain
+  # One key column of a rollup table: the tenant, the time bucket, or a plain
+  # dimension. +via+ says how to resolve it from a fact row.
+  class Dimension
+    ROLES = %i[tenant time dimension].freeze
+    TIME_GRAINS = %i[day].freeze
+
+    attr_reader :name, :path, :role, :grain
+
+    def initialize(name:, via:, role: :dimension, grain: nil, immutable: false)
+      @name = name.to_sym
+      @path = Path.parse(via)
+      @role = role.to_sym
+      @grain = grain&.to_sym
+      @immutable = immutable
+      validate!
+      freeze
+    end
+
+    def tenant?
+      role == :tenant
+    end
+
+    def time?
+      role == :time
+    end
+
+    def immutable?
+      @immutable
+    end
+
+    # A dimension resolved through an association moves fact rows into a
+    # different cell when the associated row changes, so its table needs a
+    # trigger. Declaring it immutable trades that safety for speed.
+    def watched?
+      !path.local? && !immutable?
+    end
+
+    private
+
+    def validate!
+      raise InvalidDefinitionError, "unknown dimension role #{role.inspect}" unless ROLES.include?(role)
+
+      time? ? validate_time_grain! : validate_no_grain!
+    end
+
+    def validate_time_grain!
+      return if TIME_GRAINS.include?(grain)
+
+      raise InvalidDefinitionError,
+            "time dimension #{name} needs a grain in #{TIME_GRAINS.inspect}, got #{grain.inspect}"
+    end
+
+    def validate_no_grain!
+      return if grain.nil?
+
+      raise InvalidDefinitionError, "grain only applies to a time dimension, #{name} is a #{role}"
+    end
+  end
+end

data/lib/grain/discrepancy.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Grain
+  # One cell where the rollup and its source disagree.
+  #
+  # Three kinds, and all three matter. `wrong` is the obvious one. `missing` is a
+  # cell the source has and the rollup never got. `extra` is a cell the rollup
+  # still holds after its last source row went away — the one a design built on
+  # upserts can never find, because there is nothing left to upsert against.
+  class Discrepancy
+    KINDS = %i[wrong missing extra].freeze
+
+    attr_reader :cell, :stored, :expected
+
+    def initialize(cell:, stored:, expected:)
+      @cell = cell.freeze
+      @stored = stored&.freeze
+      @expected = expected&.freeze
+      freeze
+    end
+
+    def kind
+      return :missing if stored.nil?
+      return :extra if expected.nil?
+
+      :wrong
+    end
+
+    def to_s
+      "#{kind}: #{format_cell}#{differences}"
+    end
+
+    private
+
+    def format_cell
+      cell.map { |name, value| "#{name}=#{value.inspect}" }.join(" ")
+    end
+
+    def differences
+      return "" unless kind == :wrong
+
+      changed = expected.filter_map do |measure, value|
+        "#{measure} #{stored[measure].inspect} should be #{value.inspect}" if stored[measure] != value
+      end
+      changed.empty? ? "" : " — #{changed.join(", ")}"
+    end
+  end
+end

data/lib/grain/errors.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Grain
+  # Base class for everything Grain raises, so applications can rescue Grain
+  # failures without catching unrelated errors.
+  class Error < StandardError; end
+
+  # A rollup definition is invalid: missing source, unknown dimension, a measure
+  # that cannot be maintained incrementally, and so on.
+  class InvalidDefinitionError < Error; end
+
+  # The rollup table does not match its definition — usually a definition
+  # changed without a backfill.
+  class StaleSchemaError < Error; end
+
+  # `verify` found rows where the rollup disagrees with its source.
+  class VerificationError < Error; end
+
+  # No rollup class matches the name a command was given.
+  class RollupNotFoundError < Error; end
+
+  # The change log has grown past the point where applying deltas is cheaper
+  # than a backfill.
+  class ChangeLogOverflowError < Error; end
+end

data/lib/grain/fact.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Grain
+  # The table a rollup counts rows from and reads its measures off.
+  #
+  # The model is stored by name and resolved lazily, so a definition can be
+  # loaded and inspected before the application's models are.
+  class Fact
+    attr_reader :model_name, :where
+
+    def initialize(model, where: nil)
+      @model_name = model.is_a?(Class) ? model.name.to_s : model.to_s
+      @where = where
+      validate!
+      freeze
+    end
+
+    def model
+      model_name.constantize
+    end
+
+    # A filter reaching through an association adds or removes whole rows when
+    # the associated row changes. That is not a delta on a cell, it is an
+    # invalidation of every cell the row belonged to.
+    def filtered_through_association?
+      filter_associations.any?
+    end
+
+    # The association names the filter reaches through, each of which is a table
+    # whose changes can add or remove fact rows.
+    def filter_associations
+      return [] unless where.is_a?(Hash)
+
+      where.select { |_, value| value.is_a?(Hash) }.keys.map(&:to_sym)
+    end
+
+    private
+
+    def validate!
+      raise InvalidDefinitionError, "fact needs a model" if model_name.empty?
+      return if where.nil? || where.is_a?(Hash)
+
+      raise InvalidDefinitionError, "fact where takes a hash, got #{where.class}"
+    end
+  end
+end

data/lib/grain/join_graph.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Grain
+  # The aliases and joins that reach every table a rollup needs.
+  #
+  # Registration is recursive, so two dimensions resolved through the same
+  # association share one join instead of duplicating it, and a parent is always
+  # registered — and therefore emitted — before anything reached through it.
+  class JoinGraph
+    # The fact table's alias. Measure expressions are the user's own SQL inserted
+    # verbatim, so this is the name they can qualify columns with.
+    FACT = "f"
+
+    Joined = Struct.new(:name, :model, :on, keyword_init: true)
+
+    attr_reader :definition, :types
+
+    def initialize(definition)
+      @definition = definition
+      @types = TypeResolver.new(definition)
+      @joined = {}
+      register_all
+    end
+
+    def fact_model
+      definition.fact.model
+    end
+
+    def fact_table
+      fact_model.table_name
+    end
+
+    def alias_for(hops)
+      hops.empty? ? FACT : register(hops).name
+    end
+
+    # The FROM and JOIN block, in registration order.
+    #
+    # `substitutions` maps an alias's hops to a JSON row, replacing that table with
+    # the row as it was before a change. That is how a cell a row has since left
+    # can still be found: the live table no longer points at it.
+    def from_and_joins(substitutions = {})
+      [table_expression([], FACT, substitutions)] +
+        @joined.map { |hops, joined| "JOIN #{table_expression(hops, joined.name, substitutions)} ON #{joined.on}" }
+    end
+
+    # Hop paths that reach a given table, so a change there can be traced back to
+    # the alias it arrived through.
+    def hops_for_table(table)
+      @joined.select { |_, joined| joined.model.table_name == table }.keys
+    end
+
+    private
+
+    def register_all
+      definition.key_dimensions.reject { |dimension| dimension.path.local? }
+                .each { |dimension| register(dimension.path.hops) }
+      definition.filter_associations.each { |association| register([association]) }
+    end
+
+    # A local column needs no join, so an empty path is a caller's mistake rather
+    # than a base case: treating it as one recurses forever.
+    def register(hops)
+      hops = hops.map(&:to_sym)
+      raise InvalidDefinitionError, "a local column needs no join" if hops.empty?
+
+      @joined[hops] ||= join_for(hops, parent_of(hops))
+    end
+
+    def join_for(hops, parent)
+      reflection = types.reflection!(parent.model, hops.last, hops.join("."))
+      name = "j#{@joined.size}"
+      Joined.new(name: name, model: reflection.klass, on: "#{name}.id = #{parent.name}.#{reflection.foreign_key}")
+    end
+
+    def parent_of(hops)
+      return Joined.new(name: FACT, model: fact_model, on: nil) if hops.length == 1
+
+      register(hops[0..-2])
+    end
+
+    def table_expression(hops, name, substitutions)
+      table = hops.empty? ? fact_table : @joined[hops].model.table_name
+      json = substitutions[hops]
+      return "#{table} #{name}" if json.nil?
+
+      "jsonb_populate_record(NULL::#{table}, #{ActiveRecord::Base.connection.quote(json)}::jsonb) #{name}"
+    end
+  end
+end

data/lib/grain/measure.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Grain
+  # A pre-aggregated column of a rollup table.
+  #
+  #   measure :line_count,    count: true
+  #   measure :units,         sum: "quantity", type: :bigint
+  #   measure :revenue_cents, sum: "quantity * unit_price_cents", type: :bigint
+  class Measure
+    AGGREGATES = %i[count sum min max].freeze
+
+    # count and sum can be undone by subtracting. min and max cannot: removing
+    # the current extreme means the next one has to be found in the source, so a
+    # delete forces recomputing the whole cell.
+    REVERSIBLE = %i[count sum].freeze
+
+    # A count of rows is always a whole number, so its type is not worth asking
+    # for. Every other aggregate runs over an arbitrary SQL expression whose type
+    # Grain cannot infer, and guessing would mean silently rounding somebody's
+    # revenue. The declaration is one word, and it is cheap insurance.
+    COUNT_TYPE = :bigint
+
+    attr_reader :name, :aggregate, :expression, :type
+
+    def self.from_options(name, options)
+      options = options.dup
+      type = options.delete(:type)
+      reject_ambiguous_aggregate!(name, options)
+
+      aggregate, value = options.first
+      new(name: name, aggregate: aggregate, expression: value == true ? nil : value, type: type)
+    end
+
+    def self.reject_ambiguous_aggregate!(name, options)
+      return if options.size == 1
+
+      raise InvalidDefinitionError,
+            "measure #{name} takes exactly one aggregate, got #{options.keys.inspect}"
+    end
+    private_class_method :reject_ambiguous_aggregate!
+
+    def initialize(name:, aggregate:, expression: nil, type: nil)
+      @name = name.to_sym
+      @aggregate = aggregate.to_sym
+      @expression = expression
+      @type = (type || (@aggregate == :count ? COUNT_TYPE : nil))&.to_sym
+      validate!
+      freeze
+    end
+
+    # How the measure combines when a read asks for a coarser grain than the one
+    # stored. Counts and sums add up; an extreme collapses to the extreme of the
+    # extremes, which is why min and max are storable at all despite not being
+    # reversible.
+    COARSENING = { count: :sum, sum: :sum, min: :min, max: :max }.freeze
+
+    def coarsens_with
+      COARSENING.fetch(aggregate)
+    end
+
+    # A bare count of fact rows, needing no expression to evaluate per row.
+    def counts_rows?
+      aggregate == :count && expression.nil?
+    end
+
+    # False means a delete cannot be applied as a delta and the cell has to be
+    # recomputed from the source instead.
+    def reversible?
+      REVERSIBLE.include?(aggregate)
+    end
+
+    private
+
+    def validate!
+      validate_aggregate!
+      validate_expression!
+      validate_type!
+    end
+
+    def validate_aggregate!
+      return if AGGREGATES.include?(aggregate)
+
+      raise InvalidDefinitionError,
+            "measure #{name} uses #{aggregate.inspect}, supported aggregates are #{AGGREGATES.inspect}"
+    end
+
+    def validate_expression!
+      return if aggregate == :count || !expression.nil?
+
+      raise InvalidDefinitionError, "measure #{name} aggregates with #{aggregate} and needs an expression"
+    end
+
+    def validate_type!
+      return unless type.nil?
+
+      raise InvalidDefinitionError,
+            "measure #{name} needs an explicit type, as in `#{aggregate}: ..., type: :bigint`"
+    end
+  end
+end

data/lib/grain/migration.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Grain
+  # Renders the migration that creates a rollup's table.
+  #
+  # Grain emits a migration file for the application to read and run, rather than
+  # creating tables at runtime. A rollup table is part of the schema like any
+  # other: it belongs in version control, in code review, and in schema.rb.
+  class Migration
+    # Postgres caps identifiers at 63 bytes, and a long rollup name plus a
+    # suffix passes that easily.
+    MAX_IDENTIFIER = 63
+
+    attr_reader :schema, :types
+
+    def initialize(definition)
+      @schema = Schema.new(definition)
+      @types = TypeResolver.new(definition)
+    end
+
+    def table_name
+      schema.table_name
+    end
+
+    # True when a dimension resolves to a nullable column. Postgres rejects nulls
+    # in a primary key, so the rollup falls back to a surrogate key plus a unique
+    # index that treats nulls as equal.
+    def surrogate_key?
+      types.nullable_dimensions.any?
+    end
+
+    def up
+      [create_table, uniqueness_index].compact.join("\n\n")
+    end
+
+    def down
+      "drop_table :#{table_name}"
+    end
+
+    def uniqueness_index_name
+      truncate("index_#{table_name}_uniqueness")
+    end
+
+    private
+
+    def create_table
+      lines = ["create_table :#{table_name}#{primary_key_option} do |t|"]
+      lines.concat(key_column_lines)
+      lines << ""
+      lines.concat(measure_column_lines)
+      lines << "end"
+      lines.join("\n")
+    end
+
+    # No `id: false` alongside a composite primary key. Rails accepts the pair
+    # and then creates no primary key at all, silently, which would leave the
+    # rollup table with nothing enforcing one row per cell.
+    def primary_key_option
+      return ", id: :bigint" if surrogate_key?
+
+      ", primary_key: #{schema.primary_key.inspect}"
+    end
+
+    def key_column_lines
+      schema.definition.key_dimensions.map do |dimension|
+        type = types.dimension_type(dimension)
+        nullable = types.dimension_nullable?(dimension)
+        "  t.#{type} :#{dimension.name}#{nullable ? "" : ", null: false"}"
+      end
+    end
+
+    # Measures default to zero and are never null. A null measure would poison
+    # every delta applied to it afterwards, since null plus one is null.
+    def measure_column_lines
+      schema.definition.measures.map do |measure|
+        "  t.#{types.measure_type(measure)} :#{measure.name}, null: false, default: 0"
+      end
+    end
+
+    def uniqueness_index
+      return nil unless surrogate_key?
+
+      "add_index :#{table_name}, #{schema.key_columns.inspect}, unique: true,\n" \
+        "          nulls_not_distinct: true, name: #{uniqueness_index_name.inspect}"
+    end
+
+    def truncate(identifier)
+      return identifier if identifier.bytesize <= MAX_IDENTIFIER
+
+      identifier[0, MAX_IDENTIFIER]
+    end
+  end
+end

data/lib/grain/path.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Grain
+  # A declared route from the fact table to a value: zero or more belongs_to
+  # association hops ending in a column.
+  #
+  #   Path.parse(:user_id)
+  #   # hops [], column :user_id
+  #
+  #   Path.parse(testing_section: :school_id)
+  #   # hops [:testing_section], column :school_id
+  #
+  #   Path.parse(testing_section: { assessment_window: :starts_on })
+  #   # hops [:testing_section, :assessment_window], column :starts_on
+  #
+  # Only belongs_to chains are allowed. Every fact row has to resolve to exactly
+  # one value per dimension: cross a has_many and a single row would land in
+  # several cells at once, silently doubling every count.
+  class Path
+    MAX_HOPS = 3
+
+    attr_reader :hops, :column
+
+    def self.parse(via)
+      hops, column = walk(via, [])
+      new(hops, column)
+    end
+
+    def self.walk(via, hops)
+      case via
+      when Symbol, String then [hops, via.to_sym]
+      when Hash then walk_hash(via, hops)
+      else
+        raise InvalidDefinitionError,
+              "via takes a column name or a nested hash of associations, got #{via.inspect}"
+      end
+    end
+    private_class_method :walk
+
+    def self.walk_hash(via, hops)
+      unless via.size == 1
+        raise InvalidDefinitionError,
+              "each via hop takes exactly one association, got #{via.keys.inspect}"
+      end
+
+      association, rest = via.first
+      walk(rest, hops + [association.to_sym])
+    end
+    private_class_method :walk_hash
+
+    def initialize(hops, column)
+      @hops = hops.map(&:to_sym).freeze
+      @column = column.to_sym
+      validate!
+      freeze
+    end
+
+    # A local path reads a column straight off the fact table, so no other table
+    # has to be watched to keep it correct.
+    def local?
+      hops.empty?
+    end
+
+    def root_hop
+      hops.first
+    end
+
+    def to_s
+      (hops + [column]).join(".")
+    end
+
+    def inspect
+      "#<Grain::Path #{self}>"
+    end
+
+    def ==(other)
+      other.is_a?(Path) && other.hops == hops && other.column == column
+    end
+    alias eql? ==
+
+    def hash
+      [hops, column].hash
+    end
+
+    private
+
+    def validate!
+      return if hops.size <= MAX_HOPS
+
+      raise InvalidDefinitionError, "#{self} crosses #{hops.size} associations, the limit is #{MAX_HOPS}"
+    end
+  end
+end

data/lib/grain/projection.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Grain
+  # The SQL fragments every statement needs: the expression that produces each
+  # dimension's value, the aggregate that produces each measure, and the
+  # conditions the fact's filter imposes.
+  #
+  # Reaching the tables those expressions live on is JoinGraph's job.
+  class Projection
+    FACT = JoinGraph::FACT
+
+    attr_reader :definition, :types, :joins
+
+    def initialize(definition)
+      @definition = definition
+      @types = TypeResolver.new(definition)
+      @joins = JoinGraph.new(definition)
+    end
+
+    def fact_table
+      joins.fact_table
+    end
+
+    def alias_for(hops)
+      joins.alias_for(hops)
+    end
+
+    def from_and_joins(substitutions = {})
+      joins.from_and_joins(substitutions)
+    end
+
+    def hops_for_table(table)
+      joins.hops_for_table(table)
+    end
+
+    def key_columns
+      definition.key_dimensions.map(&:name)
+    end
+
+    def measure_columns
+      definition.measures.map(&:name)
+    end
+
+    def dimension_expressions
+      definition.key_dimensions.map { |dimension| dimension_expression(dimension) }
+    end
+
+    def dimension_expression(dimension)
+      column = qualified(dimension.path)
+      dimension.time? ? bucket(column, dimension) : column
+    end
+
+    def aggregate_expressions
+      definition.measures.map { |measure| aggregate_expression(measure) }
+    end
+
+    # Cast to the measure's declared type, and not for tidiness.
+    #
+    # SUM over a bigint yields numeric, so a recompute would insert a numeric into
+    # a bigint column and let Postgres truncate it, while a verification compared
+    # the untruncated value and reported a difference. A sum over anything with a
+    # fractional part would have disagreed with itself forever, and repair could
+    # never have settled it. Computing at the stored type makes both agree.
+    def aggregate_expression(measure)
+      "#{raw_aggregate(measure)}::#{sql_type(measure.type)}"
+    end
+
+    # Conditions from the fact's filter, already qualified. Equality only in the
+    # first release; anything richer belongs in the fact's own scope.
+    def filter_conditions
+      return [] unless definition.fact.where
+
+      definition.fact.where.flat_map { |key, value| conditions_for(key, value) }
+    end
+
+    private
+
+    def raw_aggregate(measure)
+      return "COUNT(*)" if measure.counts_rows?
+
+      "COALESCE(#{measure.aggregate.to_s.upcase}(#{measure.expression}), 0)"
+    end
+
+    def qualified(path)
+      %(#{alias_for(path.hops)}."#{path.column}")
+    end
+
+    # A calendar day already is a bucket. A timestamp is not, and resolving it
+    # needs an explicit zone: left to the session, the same row would land in
+    # different buckets for different callers.
+    def bucket(column, dimension)
+      return column if types.source_type(dimension) == :date
+
+      "((#{column} AT TIME ZONE '#{Grain.config.time_zone}')::date)"
+    end
+
+    def conditions_for(key, value)
+      return ["#{FACT}.\"#{key}\" = #{quote(value)}"] unless value.is_a?(Hash)
+
+      name = alias_for([key.to_sym])
+      value.map { |column, expected| "#{name}.\"#{column}\" = #{quote(expected)}" }
+    end
+
+    def quote(value)
+      connection.quote(value)
+    end
+
+    def sql_type(type)
+      connection.type_to_sql(type)
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end