portolan 0.1.0.pre.1

data/lib/portolan/query/aggregate.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module Portolan
+  module Query
+    class Aggregate < Base
+      PARAMS = (Base::PARAMS + %w[measure group_by period]).freeze
+      MAX_GROUPS = 3
+      MAX_BUCKET_COMBINATIONS = 50
+      GROUP_LIMIT = 10_000
+      TOO_MANY_GROUPS = "more than #{GROUP_LIMIT} groups; filter or group by fewer dimensions".freeze
+
+      private
+
+      attr_reader :measure, :group_names, :group_dimensions, :time_dimension
+
+      def resolve_request
+        @group_names = parse_group_by
+        return unless dataset
+
+        @measure = dataset.measures[@params['measure']]
+        @group_dimensions = Array(group_names).filter_map { |name| dataset.dimensions[name] }
+        @time_dimension = @group_dimensions.find(&:time?)
+      end
+
+      def parse_group_by
+        case @params['group_by']
+        when nil then []
+        when ::String then [@params['group_by']]
+        when ::Array then @params['group_by'].map(&:to_s)
+        end
+      end
+
+      def validate_request
+        validate_measure
+        validate_group_by
+        validate_period
+      end
+
+      def validate_measure
+        return if measure
+
+        problem = @params['measure'] ? "unknown measure `#{@params['measure']}`" : 'measure is required'
+        errors << "#{problem}; allowed: #{Portolan.quoted(dataset.measures.keys)}"
+      end
+
+      def validate_group_by
+        return errors << 'group_by must be a list of dimension names' unless group_names
+
+        (group_names - dataset.dimensions.keys).each do |name|
+          errors << "unknown dimension `#{name}` in group_by; allowed: #{Portolan.quoted(dataset.dimensions.keys)}"
+        end
+        group_names.tally.each { |name, count| errors << "group_by names `#{name}` twice" if count > 1 }
+        errors << "group_by takes at most #{MAX_GROUPS} dimensions" if group_names.size > MAX_GROUPS
+        group_dimensions.select(&:time?).drop(1).each do |dimension|
+          errors << "group_by takes at most one date dimension; filter on `#{dimension.name}` instead"
+        end
+        validate_bucket_combinations
+      end
+
+      def validate_bucket_combinations
+        combinations = group_dimensions.select(&:buckets?).map { |dimension| dimension.buckets.size }.reduce(1, :*)
+        return if combinations <= MAX_BUCKET_COMBINATIONS
+
+        errors << "group_by makes #{combinations} bucket combinations; at most #{MAX_BUCKET_COMBINATIONS} " \
+                  'are allowed, so filter on a bucket dimension instead'
+      end
+
+      def validate_period
+        return errors << 'period only applies when grouping by a date dimension' if period && !time_dimension
+        return if time_dimension.nil? || time_dimension.periods.include?(period)
+
+        problem = period ? "period `#{period}` does not apply to" : 'period is required when grouping by'
+        errors << "#{problem} `#{time_dimension.name}`; allowed: #{Portolan.quoted(time_dimension.periods)}"
+      end
+
+      def period = @params['period']
+
+      def answer(scope)
+        rows = measured_rows(scope)
+        raise Refused, TOO_MANY_GROUPS if rows.size > GROUP_LIMIT
+
+        rows = sorted(rows)
+        Result.new(request:, rows: trimmed(rows), definitions:, total_rows: rows.size)
+      end
+
+      def measured_rows(scope)
+        bucket_dimensions, column_dimensions = group_dimensions.partition(&:buckets?)
+        rows = []
+
+        bucket_combinations(bucket_dimensions).each do |combination|
+          narrowed = combination.reduce(scope) { |current, (_dimension, bucket)| bucket.apply(current) }
+          bucket_labels = combination.to_h { |dimension, bucket| [dimension.name, bucket.name] }
+          grouped_values(narrowed, column_dimensions).each do |keys, value|
+            rows << row(bucket_labels, column_dimensions.zip(keys), value)
+          end
+          break if rows.size > GROUP_LIMIT
+        end
+        rows
+      end
+
+      def bucket_combinations(dimensions)
+        return [[]] if dimensions.empty?
+
+        lists = dimensions.map { |dimension| allowed_buckets(dimension).map { |bucket| [dimension, bucket] } }
+        lists.first.product(*lists.drop(1))
+      end
+
+      def allowed_buckets(dimension)
+        value_lists = filters.select { |filter| filter.dimension == dimension }.map(&:values)
+        names = value_lists.reduce(dimension.buckets.keys) { |kept, values| kept & values }
+        dimension.buckets.values_at(*names)
+      end
+
+      def grouped_values(scope, dimensions)
+        return [[[], measure.call(scope)]] if dimensions.empty?
+
+        grouped = dimensions.reduce(scope) do |current, dimension|
+          current = dimension.join(current)
+          current.group(group_expression(current, dimension))
+        end
+        measure.call(grouped.limit(GROUP_LIMIT + 1)).map { |keys, value| [dimensions.one? ? [keys] : keys, value] }
+      end
+
+      def group_expression(scope, dimension) = dimension.type.group(dimension.column(scope), period)
+
+      def row(bucket_labels, keyed, value)
+        labels = bucket_labels.merge(keyed.to_h { |dimension, key| [dimension.name, label(dimension, key)] })
+        grouped = group_dimensions.to_h { |dimension| [dimension.name, labels[dimension.name]] }
+        grouped.merge(measure.name => measure.label(value))
+      end
+
+      def label(dimension, key) = key.nil? ? nil : dimension.type.label(key, period)
+
+      def sorted(rows)
+        rows.sort_by { |row| [*period_order(row), -row[measure.name].to_f, row.values.map(&:to_s)] }
+      end
+
+      def period_order(row)
+        return [] unless time_dimension
+
+        label = row[time_dimension.name]
+        [label.nil? ? 0 : 1, label.to_s]
+      end
+
+      def trimmed(rows)
+        return rows.first(limit) unless time_dimension
+
+        periods = rows.chunk_while { |earlier, later| earlier[time_dimension.name] == later[time_dimension.name] }.to_a
+        kept = []
+        periods.reverse_each do |period_rows|
+          break if kept.size + period_rows.size > limit
+
+          kept = period_rows + kept
+        end
+        kept.empty? ? periods.last.to_a.first(limit) : kept
+      end
+
+      def definitions
+        definitions_for(group_dimensions).merge(measure.name => { description: measure.description })
+      end
+
+      def request = super.merge(measure: @params['measure'], group_by: Array(group_names).presence, period:)
+    end
+  end
+end

data/lib/portolan/query/base.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Portolan
+  module Query
+    class Base
+      PARAMS = %w[dataset filters limit].freeze
+      ROW_LIMIT = 50
+      UNSCOPED = %i[order select limit offset includes preload eager_load].freeze
+      INTEGER = Type::Integer.new
+
+      attr_reader :errors
+
+      def initialize(registry, params, context:)
+        @registry = registry
+        params = normalize(params)
+        @shape_error = !params.is_a?(Hash)
+        @params = @shape_error ? {} : params
+        @context = context
+        @filters = []
+        resolve
+        @errors = []
+        validate
+        @errors.freeze
+        freeze
+      end
+
+      def valid? = errors.empty?
+
+      def call
+        return Result.new(request:, errors:) unless valid?
+
+        relation = dataset.relation_for(@context).unscope(*UNSCOPED)
+        Guard.run(relation.klass) { answer(filtered(relation)) }
+      rescue Refused => e
+        Result.new(request:, errors: [e.message])
+      end
+
+      private
+
+      attr_reader :dataset, :filters
+
+      def normalize(params)
+        params = params.to_unsafe_h if params.respond_to?(:to_unsafe_h)
+        case params
+        when ::Hash then params.to_h { |key, value| [key.to_s, normalize(value)] }
+        when ::Array then params.map { |value| normalize(value) }
+        when ::Symbol then params.to_s
+        else params
+        end
+      end
+
+      def resolve
+        @dataset = @registry[@params['dataset']]
+        @filters = filter_params.map { |params| Filter.new(@dataset, params) } if @dataset
+        resolve_request
+      end
+
+      def resolve_request; end
+
+      def validate
+        validate_shape
+        validate_keys if valid?
+        validate_dataset if valid?
+        return unless valid?
+
+        validate_request
+        validate_filters
+        validate_limit
+      end
+
+      def validate_request; end
+
+      def validate_shape
+        errors << 'request must be an object' if @shape_error
+      end
+
+      def validate_keys
+        unknown = @params.keys - self.class::PARAMS
+        return if unknown.empty?
+
+        errors << "unknown key #{Portolan.quoted(unknown)}; allowed: #{Portolan.quoted(self.class::PARAMS)}"
+      end
+
+      def validate_dataset
+        return if dataset
+
+        problem = @params['dataset'] ? "unknown dataset `#{@params['dataset']}`" : 'dataset is required'
+        errors << "#{problem}; allowed: #{Portolan.quoted(@registry.names)}"
+      end
+
+      def validate_filters
+        return if @params['filters'].nil?
+        return errors << 'filters must be a list' unless @params['filters'].is_a?(::Array)
+
+        filters.each.with_index(1) do |filter, number|
+          filter.errors.each { |error| errors << "filter #{number}: #{error}" }
+        end
+      end
+
+      def validate_limit
+        errors << "limit must be an integer from 1 to #{ROW_LIMIT}" unless (1..ROW_LIMIT).cover?(limit)
+      end
+
+      def filter_params = @params['filters'].is_a?(::Array) ? @params['filters'] : []
+      def limit = @params['limit'].nil? ? ROW_LIMIT : INTEGER.cast(@params['limit'])
+
+      def filtered(relation) = filters.reduce(relation) { |current, filter| filter.apply(current) }
+
+      def definitions_for(dimensions)
+        (dimensions + filters.map(&:dimension)).uniq.to_h { |dimension| [dimension.name, dimension.definition] }
+      end
+
+      def request = { dataset: @params['dataset'] }
+    end
+  end
+end

data/lib/portolan/query/filter.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Portolan
+  module Query
+    class Filter
+      PARAMS = %w[dimension op value values].freeze
+      OPERATOR_ARITIES = {
+        'eq' => [1..1, 'one value'],
+        'gt' => [1..1, 'one value'],
+        'lt' => [1..1, 'one value'],
+        'in' => [1.., 'one or more values'],
+        'between' => [2..2, 'two values'],
+      }.freeze
+
+      attr_reader :dimension, :operator, :values, :errors
+
+      def initialize(dataset, params)
+        @dataset = dataset
+        @shape_error = !params.is_a?(Hash)
+        @params = @shape_error ? {} : params
+        @dimension = dataset.dimensions[@params['dimension']]
+        @operator = @params['op']
+        @errors = []
+        validate
+        @errors.freeze
+        @values = requested_values.map { |value| type.cast(value) }.freeze if valid?
+        freeze
+      end
+
+      def valid? = errors.empty?
+
+      def apply(scope)
+        return apply_buckets(scope) if dimension.buckets?
+
+        scope = dimension.join(scope)
+        dimension.expression ? scope.where(expression_condition(scope)) : column_condition(scope)
+      end
+
+      private
+
+      def validate
+        validate_shape
+        validate_keys if valid?
+        validate_dimension if valid?
+        validate_operator if valid?
+        validate_arity if valid?
+        validate_values if valid?
+        validate_order if valid?
+      end
+
+      def validate_shape
+        errors << 'must be an object with `dimension`, `op`, and `value` or `values`' if @shape_error
+      end
+
+      def validate_keys
+        unknown = @params.keys - PARAMS
+        errors << "unknown key #{Portolan.quoted(unknown)}; allowed: #{Portolan.quoted(PARAMS)}" if unknown.any?
+        errors << 'give `value` or `values`, not both' if @params.key?('value') && @params.key?('values')
+      end
+
+      def validate_dimension
+        return if dimension
+
+        errors << "unknown dimension `#{@params['dimension']}`; allowed: #{Portolan.quoted(@dataset.dimensions.keys)}"
+      end
+
+      def validate_operator
+        return if dimension.operators.include?(operator)
+
+        errors << "operator `#{operator}` does not apply to `#{dimension.name}`; " \
+                  "allowed: #{Portolan.quoted(dimension.operators)}"
+      end
+
+      def validate_arity
+        counts, words = OPERATOR_ARITIES.fetch(operator)
+        errors << "`#{operator}` takes #{words}" unless counts.cover?(requested_values.size)
+      end
+
+      def validate_values
+        requested_values.each do |value|
+          type.errors_for(value).each { |error| errors << "`#{dimension.name}`: #{error}" }
+        end
+      end
+
+      def validate_order
+        return unless operator == 'between'
+        return if type.cast(requested_values.first) <= type.cast(requested_values.last)
+
+        errors << '`between` needs the smaller value first'
+      end
+
+      def type = dimension.type
+      def requested_values = Array(@params.key?('values') ? @params['values'] : @params['value'])
+
+      def column_condition(scope)
+        case operator
+        when 'eq', 'in' then scope.where(dimension.name => values)
+        when 'gt' then scope.where.not(dimension.name => ..upper_bound)
+        when 'lt' then scope.where(dimension.name => ...lower_bound)
+        when 'between' then scope.where(dimension.name => lower_bound..upper_bound)
+        end
+      end
+
+      def expression_condition(scope)
+        expression = dimension.column(scope)
+        case operator
+        when 'eq', 'in' then expression.in(values.map { |value| bound(value) })
+        when 'gt' then expression.gt(bound(upper_bound))
+        when 'lt' then expression.lt(bound(lower_bound))
+        when 'between' then expression.gteq(bound(lower_bound)).and(expression.lteq(bound(upper_bound)))
+        end
+      end
+
+      def lower_bound = type.lower_bound(values.first)
+      def upper_bound = type.upper_bound(values.last)
+
+      def bound(value)
+        Arel::Nodes::BindParam.new(ActiveRecord::Relation::QueryAttribute.new(dimension.name, value, ActiveModel::Type::Value.new))
+      end
+
+      def apply_buckets(scope)
+        values.map { |name| dimension.buckets.fetch(name).apply(scope) }.reduce(:or)
+      end
+    end
+  end
+end

data/lib/portolan/query/guard.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Portolan
+  module Query
+    class Refused < StandardError; end
+
+    module Guard
+      TIMEOUT_MS = 5_000
+      TIMEOUT_ERROR = 'the query took too long; filter to a shorter period or group by fewer dimensions'
+
+      def self.run(model, timeout_ms: TIMEOUT_MS)
+        result = nil
+        outermost = !model.connection.transaction_open?
+        model.connection_class_for_self.while_preventing_writes do
+          model.transaction(requires_new: true) do
+            model.connection.execute(settings(timeout_ms, outermost))
+            result = yield
+            raise ActiveRecord::Rollback
+          end
+        end
+        result
+      rescue ActiveRecord::QueryCanceled
+        raise Refused, TIMEOUT_ERROR
+      rescue ActiveRecord::StatementInvalid => e
+        raise unless e.cause.is_a?(PG::DataException)
+
+        raise Refused, "the database could not use a value: #{e.cause.message.lines.first.sub(/\AERROR:\s*/, '').strip}"
+      end
+
+      def self.settings(timeout_ms, outermost)
+        statements = ["SET LOCAL statement_timeout = #{Integer(timeout_ms)}", "SET LOCAL TIME ZONE 'UTC'"]
+        statements << 'SET LOCAL transaction_read_only = on' if outermost
+        statements.join('; ')
+      end
+      private_class_method :settings
+    end
+  end
+end

data/lib/portolan/query/result.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Portolan
+  module Query
+    class Result
+      attr_reader :request, :rows, :definitions, :total_rows, :errors
+
+      def initialize(request:, rows: [], definitions: {}, total_rows: rows.size, errors: [])
+        @request = request.compact.freeze
+        @rows = rows.freeze
+        @definitions = definitions.freeze
+        @total_rows = total_rows
+        @errors = errors.freeze
+        freeze
+      end
+
+      def success? = errors.empty?
+      def truncated? = total_rows > rows.size
+
+      def notes
+        messages = []
+        messages << truncation_note if truncated?
+        messages << 'Periods with no rows are omitted.' if request[:period] && success?
+        messages
+      end
+
+      def to_h
+        details = { truncated: truncated?, notes:, definitions:, errors: }.compact_blank
+        success? ? { **request, rows:, **details } : { **request, **details }
+      end
+
+      private
+
+      def truncation_note
+        "Showing the #{request[:period] ? 'newest' : 'largest'} #{rows.size} of #{total_rows} rows."
+      end
+    end
+  end
+end

data/lib/portolan/registry.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Portolan
+  class Registry
+    NAME = /\A[a-z][a-z0-9_]*\z/
+
+    def initialize(datasets)
+      @datasets = build(datasets)
+      check_examples!
+      freeze
+    end
+
+    def names = @datasets.keys
+    def [](name) = @datasets[name.to_s]
+
+    def list
+      @datasets.map { |name, dataset| { name:, description: dataset.description } }
+    end
+
+    def describe(name)
+      dataset = self[name]
+      { name: name.to_s, **dataset.describe } if dataset
+    end
+
+    def check!
+      @datasets.each_value(&:check_columns!)
+      self
+    end
+
+    def aggregate(params, context:)
+      Query::Aggregate.new(self, params, context:).call
+    end
+
+    private
+
+    def build(datasets)
+      invalid!('needs a hash of names to datasets') unless datasets.is_a?(Hash) && datasets.any?
+      duplicate = datasets.keys.map(&:to_s).tally.find { |_, count| count > 1 }&.first
+      invalid!("dataset `#{duplicate}` is registered twice") if duplicate
+
+      datasets.to_h { |name, dataset| [name.to_s, checked(name.to_s, dataset)] }.freeze
+    end
+
+    def check_examples!
+      @datasets.each do |name, dataset|
+        dataset.examples.each.with_index(1) do |example, number|
+          query = Query::Aggregate.new(self, example.query.merge(dataset: name), context: nil)
+          next if query.valid?
+
+          invalid!("dataset `#{name}`: example #{number} is not a valid request: #{query.errors.join('; ')}")
+        end
+      end
+    end
+
+    def checked(name, dataset)
+      invalid!("dataset name `#{name}` must be lower case letters, digits and underscores") unless name.match?(NAME)
+      invalid!("dataset `#{name}` is not a Portolan::Dataset") unless dataset.is_a?(Class) && dataset < Dataset
+      dataset.check!
+      dataset
+    end
+
+    def invalid!(problem)
+      raise ArgumentError, "registry: #{problem}"
+    end
+  end
+end

data/lib/portolan/type/base.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Portolan
+  module Type
+    class Base
+      PERIODS = [].freeze
+
+      def operators = self.class::OPERATORS
+      def periods = self.class::PERIODS
+      def values = nil
+      def group(column, _period) = column
+      def lower_bound(value) = value
+      def upper_bound(value) = value
+      def label(value, _period = nil) = value
+    end
+  end
+end

data/lib/portolan/type/boolean.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Portolan
+  module Type
+    class Boolean < Base
+      OPERATORS = %w[eq].freeze
+
+      def name = 'boolean'
+      def errors_for(value) = [true, false, 'true', 'false'].include?(value) ? [] : ["`#{value}` is not true or false"]
+      def cast(value) = [true, 'true'].include?(value)
+    end
+  end
+end

data/lib/portolan/type/date.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Portolan
+  module Type
+    class Date < Base
+      OPERATORS = %w[gt lt between].freeze
+      PERIODS = %w[day week month quarter year].freeze
+
+      def name = 'date'
+      def errors_for(value) = cast(value)&.year&.positive? ? [] : ["`#{value}` is not a date; use YYYY-MM-DD"]
+      def group(column, period) = truncate(column, period)
+      def truncate(column, period) = Arel::Nodes::NamedFunction.new('CAST', [date_trunc(column, period).as('date')])
+      def label(value, _period = nil) = value.strftime('%Y-%m-%d')
+
+      def cast(value)
+        return unless value.is_a?(::String)
+
+        date = ::Date.strptime(value, '%Y-%m-%d')
+        date if date.strftime('%Y-%m-%d') == value
+      rescue ::Date::Error
+        nil
+      end
+
+      private
+
+      def date_trunc(column, period)
+        raise ArgumentError, "period `#{period}` does not apply to #{name}" unless periods.include?(period)
+
+        Arel::Nodes::NamedFunction.new('date_trunc', [Arel::Nodes.build_quoted(period), column])
+      end
+    end
+  end
+end

data/lib/portolan/type/date_time.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Portolan
+  module Type
+    class DateTime < Date
+      PERIODS = ['hour', *Date::PERIODS].freeze
+
+      def name = 'datetime'
+      def lower_bound(date) = ::Time.utc(date.year, date.month, date.day)
+      def upper_bound(date) = lower_bound(date).end_of_day
+      def truncate(column, period) = date_trunc(column, period)
+      def label(value, period = nil) = period == 'hour' ? value.utc.strftime('%Y-%m-%dT%H:00Z') : super(value.utc)
+    end
+  end
+end

data/lib/portolan/type/decimal.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'bigdecimal'
+
+module Portolan
+  module Type
+    class Decimal < Base
+      OPERATORS = %w[eq in gt lt between].freeze
+
+      def name = 'decimal'
+      def errors_for(value) = cast(value) ? [] : ["`#{value}` is not a number"]
+      def label(value, _period = nil) = value.to_f
+
+      def cast(value)
+        number = case value
+                 when ::Integer, ::BigDecimal then BigDecimal(value)
+                 when ::Float then BigDecimal(value.to_s)
+                 when ::String then BigDecimal(value, exception: false)
+                 end
+        number if number&.finite?
+      end
+    end
+  end
+end

data/lib/portolan/type/enum.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Portolan
+  module Type
+    class Enum < String
+      attr_reader :values
+
+      def initialize(values)
+        super()
+        @values = values.map(&:to_s).freeze
+        freeze
+      end
+
+      def errors_for(value)
+        errors = super
+        return errors if errors.any? || values.include?(value.to_s)
+
+        ["unknown value `#{value}`; allowed: #{Portolan.quoted(values)}"]
+      end
+    end
+  end
+end