grain 0.0.1

data/lib/grain/query.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Grain
+  # Reads a rollup.
+  #
+  #   OrderRevenueRollup.for(store: current_store)
+  #                     .between(1.month.ago, Date.current)
+  #                     .by(:product_id)
+  #                     .revenue_cents
+  #
+  # Narrowing returns a new query rather than changing this one, so a base query
+  # can be handed around and reused.
+  class Query
+    attr_reader :rollup, :definition
+
+    def initialize(rollup, filters: {}, range: nil, groups: {})
+      @rollup = rollup
+      @definition = rollup.definition.validate!
+      @filters = filters
+      @range = range
+      @groups = groups
+    end
+
+    # Filters on any dimension. A value may be an id, an ActiveRecord object, an
+    # array, or nil.
+    #
+    #   for(store: current_store)   # same as for(store_id: current_store.id)
+    #   for(product_id: [1, 2, 3])
+    def for(**filters)
+      merge(filters: @filters.merge(normalise(filters)))
+    end
+
+    def between(from, to = nil)
+      raise Error, "#{rollup} has no time dimension to range over" unless definition.temporal?
+
+      merge(range: to.nil? ? from : (from..to))
+    end
+
+    # Groups by dimensions, optionally coarsening the time bucket.
+    #
+    #   by(:product_id)
+    #   by(ordered_on: :month)
+    def by(*names, **coarse)
+      groups = names.to_h { |name| [dimension_name(name), nil] }
+                    .merge(coarse.transform_keys { |key| dimension_name(key) })
+      merge(groups: @groups.merge(groups))
+    end
+
+    def sql
+      QuerySql.new(definition, filters: @filters, range: @range, groups: @groups).to_s
+    end
+
+    # Every measure and ratio at once, one row per group.
+    def rows
+      @rows ||= typed_rows.map { |row| with_ratios(row) }
+    end
+
+    # Keyed by group value, or the single row when nothing is grouped.
+    def to_h
+      return rows.first || empty_row if @groups.empty?
+
+      rows.to_h { |row| [group_key(row), row] }
+    end
+
+    def value(name)
+      name = name.to_sym
+      return rows.to_h { |row| [group_key(row), row[name]] } unless @groups.empty?
+
+      (rows.first || empty_row)[name]
+    end
+
+    def respond_to_missing?(name, include_private = false)
+      readable?(name) || super
+    end
+
+    def method_missing(name, *args)
+      return value(name) if readable?(name) && args.empty?
+
+      super
+    end
+
+    private
+
+    def merge(**changes)
+      self.class.new(rollup, filters: @filters, range: @range, groups: @groups, **changes)
+    end
+
+    def readable?(name)
+      (definition.measures.map(&:name) + definition.ratios.map(&:name)).include?(name.to_sym)
+    end
+
+    def normalise(filters)
+      filters.to_h { |key, value| [dimension_name(key), identify(value)] }
+    end
+
+    # Accepts a dimension's own name, or an association-shaped one: `store` finds
+    # `store_id` when that is what the rollup is keyed on.
+    def dimension_name(key)
+      key = key.to_sym
+      names = definition.key_dimensions.map(&:name)
+      return key if names.include?(key)
+
+      suffixed = :"#{key}_id"
+      return suffixed if names.include?(suffixed)
+
+      raise Error, "#{rollup} has no dimension #{key.inspect}; it has #{names.inspect}"
+    end
+
+    def identify(value)
+      return value.map { |item| identify(item) } if value.is_a?(Array)
+
+      value.respond_to?(:id) && !value.is_a?(Numeric) ? value.id : value
+    end
+
+    # to_a hands back the driver's strings — a date arrives as "2026-08-10" —
+    # while the result object carries the adapter's own type map. Zipping the
+    # columns against cast_values applies it, so callers get Dates and Integers
+    # rather than having to parse what came out of a dashboard query.
+    def typed_rows
+      result = ActiveRecord::Base.connection.select_all(sql)
+      names = result.columns.map(&:to_sym)
+      values = result.columns.one? ? result.cast_values.map { |value| [value] } : result.cast_values
+      values.map { |row| names.zip(row).to_h }
+    end
+
+    # Ratios are never stored. They are divided here from two measures that are,
+    # so a rate is correct at whatever grain it is read at instead of frozen at
+    # the one it was computed for. A rate over nothing is nil rather than zero:
+    # there is no rate, which is not the same as a rate of none.
+    def with_ratios(row)
+      definition.ratios.each_with_object(row) do |ratio, result|
+        denominator = result[ratio.denominator].to_f
+        result[ratio.name] = denominator.zero? ? nil : result[ratio.numerator] / denominator
+      end
+    end
+
+    def empty_row
+      definition.measures.to_h { |measure| [measure.name, measure.coarsens_with == :sum ? 0 : nil] }
+                .merge(definition.ratios.to_h { |ratio| [ratio.name, nil] })
+    end
+
+    def group_key(row)
+      keys = @groups.keys.map { |name| row[name] }
+      keys.length == 1 ? keys.first : keys
+    end
+  end
+end

data/lib/grain/query_sql.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Grain
+  # Builds the statement a read runs.
+  #
+  # Any dimension left out of the grouping is aggregated away, which is the
+  # property the whole design rests on: a day rolls up into a month by addition,
+  # so one stored grain answers questions at every coarser one.
+  class QuerySql
+    COARSER = %i[day week month quarter year].freeze
+
+    attr_reader :definition
+
+    def initialize(definition, filters: {}, range: nil, groups: {})
+      @definition = definition
+      @filters = filters
+      @range = range
+      @groups = groups
+    end
+
+    def to_s
+      <<~SQL.strip
+        SELECT #{selection.join(", ")}
+        FROM #{definition.table_name}#{where_clause}#{group_clause}#{order_clause}
+      SQL
+    end
+
+    private
+
+    def selection
+      @groups.map { |name, coarse| "#{group_expression(name, coarse)} AS #{quote_column(name)}" } +
+        definition.measures.map { |measure| "#{measure_expression(measure)} AS #{measure.name}" }
+    end
+
+    # A sum over nothing is zero, so it is coalesced. An extreme over nothing is
+    # left null on purpose: there is no largest value, which is not the same as a
+    # largest value of zero.
+    #
+    # Cast back to the measure's declared type for the same reason the write side
+    # does: SUM over a bigint yields numeric, and without this a column stored as
+    # an integer would read back as a BigDecimal.
+    def measure_expression(measure)
+      aggregate = "#{measure.coarsens_with.to_s.upcase}(#{quote_column(measure.name)})"
+      aggregate = "COALESCE(#{aggregate}, 0)" if measure.coarsens_with == :sum
+      "#{aggregate}::#{connection.type_to_sql(measure.type)}"
+    end
+
+    def group_expression(name, coarse)
+      column = quote_column(name)
+      return column if coarse.nil?
+      raise Error, "unknown grain #{coarse.inspect}, expected one of #{COARSER.inspect}" unless COARSER.include?(coarse)
+
+      "(DATE_TRUNC('#{coarse}', #{column})::date)"
+    end
+
+    def where_clause
+      conditions = @filters.map { |name, value| condition_for(name, value) } + [range_condition].compact
+      conditions.empty? ? "" : "\nWHERE #{conditions.join(" AND ")}"
+    end
+
+    def condition_for(name, value)
+      column = quote_column(name)
+      case value
+      when nil then "#{column} IS NULL"
+      when Array then "#{column} IN (#{value.map { |item| quote(item) }.join(", ")})"
+      else "#{column} = #{quote(value)}"
+      end
+    end
+
+    def range_condition
+      return nil if @range.nil?
+
+      "#{quote_column(definition.time.name)} BETWEEN #{quote(@range.first)} AND #{quote(@range.last)}"
+    end
+
+    def group_clause
+      @groups.empty? ? "" : "\nGROUP BY #{group_positions}"
+    end
+
+    def order_clause
+      @groups.empty? ? "" : "\nORDER BY #{group_positions}"
+    end
+
+    def group_positions
+      (1..@groups.length).to_a.join(", ")
+    end
+
+    def quote_column(name)
+      connection.quote_column_name(name)
+    end
+
+    def quote(value)
+      connection.quote(value)
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end

data/lib/grain/railtie.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+
+module Grain
+  # Hooks Grain into a Rails application: rake tasks, autoloading of
+  # app/rollups, and the default logger.
+  class Railtie < ::Rails::Railtie
+    initializer "grain.logger" do
+      Grain.config.logger ||= Rails.logger
+    end
+
+    rake_tasks do
+      load File.expand_path("../tasks/grain.rake", __dir__)
+    end
+
+    initializer "grain.autoload_rollups" do |app|
+      app.config.autoload_paths << app.root.join("app/rollups") if app.root.join("app/rollups").exist?
+    end
+  end
+end

data/lib/grain/ratio.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Grain
+  # A derived value kept as its two parts and divided on read.
+  #
+  # Ratios are never stored pre-divided: averaging averages is wrong, and a
+  # stored rate cannot be rolled up from day to month. Numerator and denominator
+  # both sum, so their quotient stays correct at every grain.
+  class Ratio
+    attr_reader :name, :numerator, :denominator
+
+    def initialize(name:, numerator:, denominator:)
+      @name = name.to_sym
+      @numerator = numerator.to_sym
+      @denominator = denominator.to_sym
+      validate!
+      freeze
+    end
+
+    private
+
+    def validate!
+      return unless numerator == denominator
+
+      raise InvalidDefinitionError, "ratio #{name} divides #{numerator.inspect} by itself"
+    end
+  end
+end

data/lib/grain/recompute.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Grain
+  # Rebuilds cells from the source: the primitive everything else falls back to,
+  # and the only operation that is correct no matter what happened.
+  #
+  # Delete and then re-insert rather than upsert, because a cell can legitimately
+  # become empty. An upsert would leave the old numbers standing when the last
+  # source row for a cell goes away.
+  #
+  # Being complete rather than incremental has a useful consequence: a recompute
+  # cannot be half-applied and cannot be applied out of order, so a backfill and
+  # the worker can run at the same time without coordinating.
+  class Recompute
+    attr_reader :definition, :projection
+
+    def initialize(definition)
+      @definition = definition
+      @projection = Projection.new(definition)
+    end
+
+    # Rebuilds a known list of cells.
+    def call(cells)
+      cells = cells.uniq
+      return 0 if cells.empty?
+
+      rebuild(matches(cells) { |key| quote_column(key) }, matches(cells) { |key| expression_for(key) })
+      cells.length
+    end
+
+    # Rebuilds every cell in a slice, for when the cells are not known in advance
+    # and all that is known is which slice has to be right — a backfill working
+    # through one day or one tenant at a time.
+    def call_slice(dimension, value)
+      rebuild(
+        "#{quote_column(dimension.name)} IS NOT DISTINCT FROM #{quote(value)}",
+        "#{expression_for(dimension.name)} IS NOT DISTINCT FROM #{quote(value)}"
+      )
+    end
+
+    private
+
+    def rebuild(stored_where, source_where)
+      connection.execute("DELETE FROM #{table_name} WHERE #{stored_where}")
+      connection.execute(insert_sql(source_where))
+    end
+
+    def insert_sql(source_where)
+      <<~SQL
+        INSERT INTO #{table_name} (#{columns.join(", ")})
+        SELECT #{expressions.join(", ")}
+        FROM #{projection.from_and_joins.join(" ")}
+        WHERE #{(projection.filter_conditions + [source_where]).join(" AND ")}
+        GROUP BY #{projection.dimension_expressions.join(", ")}
+      SQL
+    end
+
+    def table_name
+      definition.table_name
+    end
+
+    def columns
+      projection.key_columns + projection.measure_columns
+    end
+
+    def expressions
+      projection.dimension_expressions + projection.aggregate_expressions
+    end
+
+    def matches(cells, &naming)
+      "(#{cells.map { |cell| match(cell, &naming) }.join(" OR ")})"
+    end
+
+    # IS NOT DISTINCT FROM rather than =, because a dimension resolved from a
+    # nullable column has null as a legitimate coordinate, and null = null is
+    # never true.
+    def match(cell)
+      parts = projection.key_columns.map do |key|
+        "#{yield(key)} IS NOT DISTINCT FROM #{quote(cell[key])}"
+      end
+      "(#{parts.join(" AND ")})"
+    end
+
+    def expression_for(key)
+      @expressions ||= projection.key_columns.zip(projection.dimension_expressions).to_h
+      @expressions.fetch(key)
+    end
+
+    def quote_column(key)
+      connection.quote_column_name(key)
+    end
+
+    def quote(value)
+      connection.quote(value)
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end

data/lib/grain/registry.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Grain
+  # Which rollups exist, and which of them care about a given table.
+  #
+  # A trigger records only that a row in some table changed, so the worker needs
+  # the reverse mapping: table to the rollups that read it, and in what capacity.
+  # A table can be a rollup's fact, one of the tables it resolves dimensions
+  # through, or both.
+  module Registry
+    class << self
+      # Rollups are found rather than self-registered: with autoloading, a class
+      # that nobody has referenced yet does not exist, so a registry populated by
+      # `inherited` would be empty in exactly the process that needs it.
+      def all
+        @all ||= load_rollups
+      end
+
+      def reset!
+        @all = nil
+      end
+
+      def register(*rollups)
+        (@all ||= []).concat(rollups).uniq!
+        all
+      end
+
+      # Rollups whose fact table this is: a change here is a change to what is
+      # being counted.
+      def facts_for(table)
+        all.select { |rollup| fact_table(rollup) == table }
+      end
+
+      # Rollups that resolve a dimension or a filter through this table: a change
+      # here can move fact rows between cells without touching the facts.
+      def watchers_for(table)
+        all.select { |rollup| watched_tables(rollup).include?(table) }
+      end
+
+      def for_table(table)
+        (facts_for(table) + watchers_for(table)).uniq
+      end
+
+      private
+
+      def load_rollups
+        eager_load_rollups
+        Rollup.subclasses.select { |rollup| rollup.name && valid?(rollup) }
+      end
+
+      def eager_load_rollups
+        return unless defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+
+        Dir[Rails.root.join("app/rollups/**/*.rb")].sort.each { |path| require path }
+      end
+
+      # A rollup that cannot be used is left out rather than allowed to take the
+      # whole worker down with it, but never silently: everything else it shares
+      # a log with would go stale while the log kept draining.
+      def valid?(rollup)
+        rollup.definition.validate!
+        rollup.definition.fact.model
+        true
+      rescue InvalidDefinitionError, NameError => e
+        warn_about(rollup, e)
+        false
+      end
+
+      def warn_about(rollup, error)
+        message = "Grain is skipping #{rollup}: #{error.message}"
+        logger = Grain.config.logger
+        logger ? logger.warn(message) : Kernel.warn(message)
+      end
+
+      def fact_table(rollup)
+        rollup.definition.fact.model.table_name
+      end
+
+      def watched_tables(rollup)
+        WatchedColumns.new(rollup.definition).to_h.keys
+      end
+    end
+  end
+end

data/lib/grain/rollup.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Grain
+  # Base class for rollup definitions. A subclass declares the shape of one
+  # pre-aggregated table. Nothing here touches the database.
+  #
+  #   class AssessmentResultRollup < Grain::Rollup
+  #     fact TestingSectionStudent, where: { user: { role: "student" } }
+  #
+  #     tenant    :school_id, via: { testing_section: :school_id }
+  #     time      :assessed_on,
+  #               via: { testing_section: { assessment_window: :starts_on } },
+  #               grain: :day
+  #     dimension :grade_id,  via: { testing_section: :grade_id }
+  #     dimension :window_id, via: { testing_section: :assessment_window_id },
+  #               immutable: true
+  #
+  #     measure :attempts,     count: true
+  #     measure :passed_count, sum: "CASE WHEN score >= 60 THEN 1 ELSE 0 END"
+  #     ratio   :pass_rate, of: :passed_count, over: :attempts
+  #   end
+  class Rollup
+    class << self
+      def definition
+        @definition ||= Definition.new(self)
+      end
+
+      def fact(model, where: nil)
+        definition.declare_fact(model, where: where)
+      end
+
+      def tenant(name, via:)
+        definition.add_dimension(name, via: via, role: :tenant)
+      end
+
+      def time(name, via:, grain:)
+        definition.add_dimension(name, via: via, role: :time, grain: grain)
+      end
+
+      def dimension(name, via:, immutable: false)
+        definition.add_dimension(name, via: via, immutable: immutable)
+      end
+
+      def measure(name, **options)
+        definition.add_measure(name, options)
+      end
+
+      def ratio(name, of:, over:)
+        definition.add_ratio(name, numerator: of, denominator: over)
+      end
+
+      def table_name
+        definition.table_name
+      end
+
+      def validate!
+        definition.validate!
+      end
+
+      def query
+        Query.new(self)
+      end
+
+      # Reading entry points. Each returns a query that can be narrowed further.
+      def for(**filters)
+        query.for(**filters)
+      end
+
+      def between(from, to = nil)
+        query.between(from, to)
+      end
+
+      def by(*names, **coarse)
+        query.by(*names, **coarse)
+      end
+
+      # Populates the rollup from data that already exists. A new rollup is empty
+      # until this runs: its triggers only see what happens next.
+      def backfill(from: nil, pause: 0, &progress)
+        Backfill.new(self, from: from, pause: pause).call(&progress)
+      end
+
+      # Recomputes from the source and reports every cell that disagrees. Pass
+      # repair: true to rebuild the ones that do.
+      def verify(tenant: nil, between: nil, repair: false)
+        Verification.new(self, tenant: tenant, between: between).call(repair: repair)
+      end
+    end
+  end
+end

data/lib/grain/rollup_lookup.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Grain
+  # Finds a rollup class from the name given on a command line, accepting it with
+  # or without the Rollup suffix.
+  #
+  # Kept out of the generator so the rule is testable on its own: Thor catches the
+  # errors a generator raises and prints them rather than letting them out, which
+  # makes assertions about failure unreliable at that level.
+  module RollupLookup
+    class << self
+      def candidates(name)
+        base = name.to_s.camelize
+        [base, "#{base.delete_suffix("Rollup")}Rollup"].uniq
+      end
+
+      def find(name)
+        candidates(name).filter_map(&:safe_constantize).find { |constant| rollup?(constant) }
+      end
+
+      def find!(name)
+        find(name) || raise(RollupNotFoundError, message_for(name))
+      end
+
+      def rollup?(constant)
+        constant.is_a?(Class) && constant < Rollup ? true : false
+      end
+
+      def message_for(name)
+        "Could not find a Grain::Rollup named #{candidates(name).join(" or ")}. " \
+          "Run `rails generate grain:rollup #{suggested_argument(name)}` first."
+      end
+
+      private
+
+      def suggested_argument(name)
+        name.to_s.underscore.delete_suffix("_rollup")
+      end
+    end
+  end
+end

data/lib/grain/schema.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Grain
+  # The shape of the physical table a rollup is stored in, derived from its
+  # definition: column names, their order, and the primary key.
+  #
+  # Column types are not here on purpose. A dimension's type has to match the
+  # source column it is resolved from, which means reading the database. That
+  # belongs to the migration generator; this class stays pure so the shape can be
+  # reasoned about and tested without a connection.
+  class Schema
+    attr_reader :definition
+
+    def initialize(definition)
+      @definition = definition.validate!
+      freeze
+    end
+
+    def table_name
+      definition.table_name
+    end
+
+    # Key columns in a fixed order: tenant, then the time bucket if the rollup
+    # has one, then the plain dimensions in declaration order.
+    #
+    # The order is part of the contract, not an implementation detail. Reads
+    # filter on a prefix of it (tenant, or tenant and a date range), and so does
+    # every scoped recompute, so both ride the primary key index.
+    def key_columns
+      definition.key_dimensions.map(&:name)
+    end
+
+    def measure_columns
+      definition.measures.map(&:name)
+    end
+
+    # Every column of the rollup table.
+    #
+    # Ratios are absent by design: they are divided on read from two measures
+    # that are stored, so a rate stays correct at every grain instead of being
+    # frozen at the one it was computed for.
+    def columns
+      key_columns + measure_columns
+    end
+
+    def primary_key
+      key_columns
+    end
+
+    # No secondary indexes in the first release, and that is a decision rather
+    # than an omission. Every index slows down the writes that keep the rollup
+    # fresh, and Grain cannot know an application's read patterns; the primary
+    # key already covers the dominant ones. Extra indexes are the application's
+    # call, added in its own migration.
+    def indexes
+      []
+    end
+
+    def temporal?
+      definition.temporal?
+    end
+  end
+end