hamster-web 0.1.0.pre.2

data/lib/hamster/data/read/nodes.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Hamster
+  module Read
+    class Error < StandardError; end
+    class ValidationError < Error; end
+    class DecodeError < Error; end
+
+    TYPES = { integer: 23, bigint: 20, text: 25, boolean: 16, timestamptz: 1184 }.freeze
+
+    def self.snapshot(value)
+      case value
+      when Array then value.map { |item| snapshot(item) }.freeze
+      when Hash
+        result = {}
+        value.each { |key, item| result[snapshot(key)] = snapshot(item) }
+        result.freeze
+      when String, Time then value.dup.freeze
+      else value
+      end
+    end
+
+    def self.structural(value)
+      case value
+      when Node, Source then value.key
+      when Array then value.map { |item| structural(item) }
+      when Hash
+        result = {}
+        value.each { |key, item| result[key] = structural(item) }
+        result
+      else value
+      end
+    end
+
+    def self.identifier(value)
+      name = value.to_s
+      raise ValidationError, "Invalid identifier #{name.inspect}" if name.empty? || name.include?("\0")
+      name.dup.freeze
+    end
+
+    class Source
+      attr_reader :row_class, :name, :schema, :label, :columns
+      def initialize(row_class, label = nil)
+        @row_class = row_class
+        @name = row_class.table_name
+        @schema = row_class.schema_name
+        @label = Read.identifier(label || @name)
+        @columns = Read.snapshot(row_class.columns)
+        freeze
+      end
+
+      def [](name)
+        Node.new(:column, [self, name.to_sym])
+      end
+
+      def all
+        Relation.new(self)
+      end
+
+      def key
+        [row_class, schema, name, label]
+      end
+    end
+
+    # One expression representation is shared by validation, compilation and decoding.
+    class Node
+      attr_reader :kind, :args, :options
+      def initialize(kind, args = [], options = {})
+        @kind = kind
+        @args = Read.snapshot(args)
+        @options = Read.snapshot(options)
+        freeze
+      end
+
+      def key
+        [kind, Read.structural(args), Read.structural(options)]
+      end
+
+      def eq(value); compare(:eq, value); end
+      def not_eq(value); compare(:not_eq, value); end
+      def gt(value); compare(:gt, value); end
+      def gte(value); compare(:gte, value); end
+      def lt(value); compare(:lt, value); end
+      def lte(value); compare(:lte, value); end
+      def ilike(pattern); compare(:ilike, pattern); end
+      def in(values); Node.new(:in, [self, values]); end
+      def not_in(values); Node.new(:not_in, [self, values]); end
+      def as(name); Node.new(:as, [self, Read.identifier(name)]); end
+      def asc; Node.new(:order, [self, :asc]); end
+      def desc; Node.new(:order, [self, :desc]); end
+      def and(other); Node.new(:and, [self, other]); end
+      def or(other); Node.new(:or, [self, other]); end
+      def not; Node.new(:not, [self]); end
+
+      def compare(operator, value)
+        Node.new(operator, [self, value.is_a?(Node) ? value : Node.new(:value, [value])])
+      end
+    end
+  end
+
+  module SQL
+    def self.count(column = nil, distinct: false)
+      Read::Node.new(:count, column.nil? ? [] : [column], { distinct: distinct })
+    end
+
+    def self.output(name)
+      Read::Node.new(:output, [Read.identifier(name)])
+    end
+
+    def self.bind(value)
+      Read::Node.new(:value, [value])
+    end
+
+    def self.raw(parts, dependencies:, type:, nullable:, aggregate: false)
+      Read::Node.new(:raw, parts, { dependencies: dependencies, type: type, nullable: nullable, aggregate: aggregate })
+    end
+  end
+end

data/lib/hamster/data/read/relation.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module Hamster
+  class Relation
+    attr_reader :source, :state
+
+    def initialize(source, state = {})
+      @source = source
+      @state = Read.snapshot({ select: [], explicit: false, joins: [], where: [], group: [],
+                              having: [], order: [], distinct: false, limit: nil, offset: nil, page: nil,
+                              identity: "anonymous" }.merge(state))
+      freeze
+    end
+
+    def all; self; end
+    def named(identity); change(identity: identity.to_s); end
+    def select(*items); change(select: state[:select] + expressions(items), explicit: true); end
+    def reselect(*items); change(select: expressions(items), explicit: true); end
+    def where(condition); change(where: state[:where] + conditions(condition)); end
+    def having(condition); change(having: state[:having] + conditions(condition)); end
+    def group(*items); change(group: state[:group] + expressions(items)); end
+    def order(*items); change(order: state[:order] + orderings(items)); end
+    def reorder(*items); change(order: orderings(items)); end
+    def distinct(enabled = true); change(distinct: enabled); end
+    def limit(value); change(limit: value); end
+    def offset(value); change(offset: value, page: nil); end
+
+    def page(number)
+      unless number.is_a?(Integer) && number > 0
+        raise Read::ValidationError, "#{state[:identity]}: PAGE: use a positive integer starting at 1"
+      end
+      change(page: number, offset: nil)
+    end
+
+    def joins(other, on:)
+      join(other, on, :inner)
+    end
+
+    def left_joins(other, on:)
+      join(other, on, :left)
+    end
+
+    def compile
+      Read::Compiler.new(Read::Validator.new(resolve_page).validate).compile
+    end
+
+    def to_sql; compile.sql; end
+    def binds; compile.binds; end
+
+    def load(connection: nil)
+      Instrumentation.instrument("query.load", attributes: {"query.name" => state[:identity]}) do
+        compiled = compile
+        result = (connection || Record.connection).execute(compiled.sql, compiled.binds)
+        Read::Decoder.new(compiled).decode(result)
+      end
+    end
+
+    def count(connection: nil)
+      Instrumentation.instrument("query.count", attributes: {"query.name" => state[:identity]}) do
+        compiled = Read::Compiler.new(Read::Validator.new(resolve_page).validate).compile(count: true)
+        result = (connection || Record.connection).execute(compiled.sql, compiled.binds)
+        rows = Read::Decoder.new(compiled).decode(result)
+        raise Read::DecodeError, "Count result must contain exactly one row" unless rows.length == 1
+        rows.first[:count]
+      end
+    end
+
+    # Terminal reads stay lazy until this call and use PostgreSQL for the limit.
+    def first(connection: nil)
+      return resolve_page.first(connection: connection) unless state[:page].nil?
+      compile
+      return nil if state[:limit] == 0
+
+      query = self
+      if query.state[:order].empty? && implicit_primary_order?
+        query = query.order(source[:id].asc)
+      end
+      query.limit(query.state[:limit].nil? ? 1 : [query.state[:limit], 1].min)
+           .load(connection: connection).first
+    end
+
+    def last(connection: nil)
+      return resolve_page.last(connection: connection) unless state[:page].nil?
+      compile
+      return nil if state[:limit] == 0
+
+      # Reversing an ordered, offset relation changes which rows the offset skips.
+      # Load that already-scoped window to preserve its meaning.
+      if !state[:offset].nil? || (state[:order].empty? && !implicit_primary_order?)
+        query = state[:order].empty? && implicit_primary_order? ? order(source[:id].asc) : self
+        return query.load(connection: connection).last
+      end
+
+      if state[:order].empty? && !state[:limit].nil?
+        return order(source[:id].asc).load(connection: connection).last
+      end
+
+      reversed = if state[:order].empty?
+                   order(source[:id].desc)
+                 else
+                   order = state[:order].map do |item|
+                     direction = item.args[1] == :asc ? :desc : :asc
+                     Read::Node.new(:order, [item.args[0], direction])
+                   end
+                   reorder(*order)
+                 end
+      if reversed.state[:limit].nil?
+        reversed.limit(1).load(connection: connection).first
+      else
+        reversed.load(connection: connection).first
+      end
+    end
+
+    private
+
+    def validate_page_size!
+      unless state[:limit].is_a?(Integer) && state[:limit] > 0
+        raise Read::ValidationError, "#{state[:identity]}: PAGE: pagination requires a positive page size; add .limit(size)"
+      end
+    end
+
+    def resolve_page
+      return self if state[:page].nil?
+      validate_page_size!
+      query = self
+      if state[:order].empty?
+        unless implicit_primary_order?
+          raise Read::ValidationError, "#{state[:identity]}: PAGE: add an explicit order for grouped, distinct, aggregate, or id-less queries"
+        end
+        query = order(source[:id].asc)
+      end
+      query.offset((state[:page] - 1) * state[:limit])
+    end
+
+    def change(changes)
+      Relation.new(source, state.merge(changes))
+    end
+
+    def expression(item)
+      item.is_a?(Symbol) ? source[item] : item
+    end
+
+    def expressions(items)
+      items.flatten.map { |item| expression(item) }
+    end
+
+    def conditions(condition)
+      if condition.is_a?(Hash)
+        condition.flat_map do |key, value|
+          column = source[key]
+          if value.is_a?(Range)
+            bounds = []
+            bounds << column.gte(value.begin) unless value.begin.nil?
+            if !value.end.nil?
+              bounds << (value.exclude_end? ? column.lt(value.end) : column.lte(value.end))
+            end
+            bounds
+          elsif value.is_a?(Array)
+            [column.in(value)]
+          else
+            [column.eq(value)]
+          end
+        end
+      else
+        [condition]
+      end
+    end
+
+    def orderings(items)
+      items.flatten.flat_map do |item|
+        if item.is_a?(Hash)
+          item.map { |key, direction| Read::Node.new(:order, [source[key], direction]) }
+        else
+          node = expression(item)
+          [node.is_a?(Read::Node) && node.kind == :order ? node : Read::Node.new(:order, [node, :asc])]
+        end
+      end
+    end
+
+    def implicit_primary_order?
+      id = source.columns[:id]
+      id && id[:enabled] && state[:group].empty? && state[:having].empty? && !state[:distinct] &&
+        !state[:select].any? { |item| aggregate_expression?(item) } &&
+        !state[:order].any? { |item| aggregate_expression?(item) }
+    end
+
+    def aggregate_expression?(item)
+      return false unless item.is_a?(Read::Node)
+      return true if item.kind == :count || (item.kind == :raw && item.options[:aggregate])
+      item.args.any? { |argument| aggregate_expression?(argument) }
+    end
+
+    def join(other, on, kind)
+      joined = other.is_a?(Read::Source) ? other : other.all.source
+      change(joins: state[:joins] + [Read::Node.new(:join, [joined, on], { kind: kind })])
+    end
+  end
+end

data/lib/hamster/data/read/schema.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Hamster
+  module Read
+    class Schema
+      def self.validate(connection, tables = Table.registry)
+        errors = []
+        tables.each do |table|
+          catalog = connection.execute(<<~SQL, [table.schema_name, table.table_name]).hashes
+            SELECT a.attname AS name, a.atttypid::text AS type_id,
+                   a.attnotnull::text AS not_null
+            FROM pg_catalog.pg_attribute a
+            JOIN pg_catalog.pg_class c ON c.oid = a.attrelid
+            JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+            WHERE n.nspname = $1 AND c.relname = $2
+              AND c.relkind IN ('r', 'p', 'v', 'm', 'f')
+              AND a.attnum > 0 AND NOT a.attisdropped
+          SQL
+          prefix = "#{table.schema_name}.#{table.table_name}"
+          errors << "#{prefix}: table is missing" if catalog.empty?
+          table.columns.each do |name, definition|
+            next unless definition[:enabled]
+            actual = catalog.find { |row| row["name"] == name.to_s }
+            if actual.nil?
+              errors << "#{prefix}.#{name}: column is missing"
+            else
+              unless actual["type_id"] == TYPES[definition[:type]].to_s
+                errors << "#{prefix}.#{name}: incompatible type (expected #{definition[:type]})"
+              end
+              if !definition[:nullable] && actual["not_null"] != "true"
+                errors << "#{prefix}.#{name}: database permits NULL"
+              end
+            end
+          end
+        end
+        errors
+      end
+    end
+  end
+end

data/lib/hamster/data/read/validator.rb

@@ -0,0 +1,273 @@
+# frozen_string_literal: true
+
+
+module Hamster
+  module Read
+    class Validator
+      attr_reader :relation, :source, :state, :outputs, :dependencies, :table_rows, :result_class
+
+      def initialize(relation)
+        @relation = relation
+        @source = relation.source
+        @state = relation.state.dup
+        @outputs = []
+        @dependencies = []
+        @sources = [source]
+        @left_sources = []
+        @aliases = {}
+      end
+
+      def fail!(message, clause)
+        raise ValidationError, "#{state[:identity]}: #{clause}: #{message}"
+      end
+
+      def validate
+        state[:joins].each do |join|
+          other = join.args[0]
+          fail!("duplicate source alias #{other.label}", "JOIN") if @sources.any? { |s| s.label == other.label }
+          @sources << other
+          @left_sources << other.key if join.options[:kind] == :left
+        end
+        selections = if state[:explicit]
+                       state[:select]
+                     else
+                       source.columns.select { |_key, spec| spec[:enabled] }.map { |key, _spec| source[key] }
+                     end
+        fail!("empty projection", "SELECT") if selections.empty?
+        (state[:declared_fields] || []).each { |node| @aliases[node.args[1]] = node.args[0] }
+        declared_aliases = @aliases.dup
+        declared_projection = (state[:declared_fields] || []).map do |node|
+          resolved = resolve(node, "SELECT", [])
+          [output_name(resolved), unwrap(resolved).key]
+        end
+        selections.each do |node|
+          if node.is_a?(Node) && node.kind == :as
+            name = node.args[1]
+            # Query output references continue to name the declared field even
+            # when a runtime projection replaces or aliases that field.
+            @aliases[name] = node.args[0] unless declared_aliases.key?(name)
+          end
+        end
+        state[:select] = unique(selections.map do |node|
+          resolved = resolve(node, "SELECT", [])
+          node.kind == :output ? resolved.as(node.args[0]) : resolved
+        end)
+        state[:group] = unique(state[:group].map { |node| resolve(node, "GROUP BY", []) })
+        [:where, :having, :order].each do |clause|
+          state[clause] = state[clause].map { |node| resolve(node, clause.to_s.upcase, []) }
+        end
+        visible = [source]
+        state[:joins] = state[:joins].map do |join|
+          visible << join.args[0]
+          node = resolve(join.args[1], "ON", [])
+          info = inspect_node(node, "ON", visible)
+          predicate!(info, "ON")
+          Node.new(:join, [join.args[0], node], join.options)
+        end
+        state[:select].each do |node|
+          info = inspect_node(node, "SELECT", @sources)
+          name = output_name(node)
+          fail!("projection needs an explicit alias", "SELECT") unless name
+          fail!("conflicting output #{name}", "SELECT") if outputs.any? { |out| out[:name] == name }
+          outputs << { name: name, type: info[:type], nullable: info[:nullable], expression: unwrap(node) }.freeze
+        end
+        [:where, :having].each do |clause|
+          state[clause].each { |node| predicate!(inspect_node(node, clause.to_s.upcase, @sources), clause.to_s.upcase) }
+        end
+        state[:group].each { |node| inspect_node(node, "GROUP BY", @sources) }
+        state[:order].each do |node|
+          fail!("unsupported ordering", "ORDER BY") unless node.is_a?(Node) && node.kind == :order
+          fail!("direction must be :asc or :desc", "ORDER BY") unless [:asc, :desc].include?(node.args[1])
+          inspect_node(node.args[0], "ORDER BY", @sources)
+        end
+        grouped = !state[:group].empty? || !state[:having].empty? ||
+                  state[:select].any? { |node| aggregate?(node) } || state[:order].any? { |node| aggregate?(node.args[0]) }
+        if grouped
+          state[:select].each { |node| check_group(unwrap(node), "SELECT") }
+          state[:having].each { |node| check_group(node, "HAVING") }
+          state[:order].each { |node| check_group(node.args[0], "ORDER BY") }
+        end
+        fail!("distinct must be boolean", "DISTINCT") unless [true, false].include?(state[:distinct])
+        if state[:distinct]
+          state[:order].each do |node|
+            unless outputs.any? { |out| out[:expression].key == node.args[0].key }
+              fail!("DISTINCT ordering must be selected", "ORDER BY")
+            end
+          end
+        end
+        [:limit, :offset].each do |clause|
+          value = state[clause]
+          unless value.nil? || (value.is_a?(Integer) && value >= 0)
+            fail!("must be a nonnegative integer", clause.to_s.upcase)
+          end
+        end
+        @table_rows = state[:joins].empty? && !grouped && state[:select].all? do |node|
+          node.kind == :column && node.args[0].key == source.key
+        end
+        if state[:result_class]
+          @result_class = state[:result_class] if outputs.all? do |out|
+            declared_projection.include?([out[:name], out[:expression].key])
+          end
+          @table_rows = false
+        end
+        @state = Read.snapshot(state)
+        @outputs.freeze
+        @dependencies.freeze
+        self
+      end
+
+      def resolve(node, clause, stack)
+        fail!("unsupported expression #{node.class}", clause) unless node.is_a?(Node)
+        if node.kind == :output
+          name = node.args[0]
+          fail!("unknown output #{name}", clause) unless @aliases.key?(name)
+          fail!("cyclic output #{name}", clause) if stack.include?(name)
+          return resolve(@aliases[name], clause, stack + [name])
+        end
+        args = node.args.map { |arg| arg.is_a?(Node) ? resolve(arg, clause, stack) : arg }
+        Node.new(node.kind, args, node.options)
+      end
+
+      def unique(nodes)
+        result = []
+        nodes.each { |node| result << node unless result.any? { |old| old.key == node.key } }
+        result
+      end
+
+      def unwrap(node)
+        node.kind == :as ? node.args[0] : node
+      end
+
+      def output_name(node)
+        return node.args[1] if node.kind == :as
+        return node.args[1].to_s if node.kind == :column
+        return "count" if node.kind == :count
+        nil
+      end
+
+      def info(type, nullable, aggregate = false)
+        { type: type, nullable: nullable, aggregate: aggregate }
+      end
+
+      def predicate!(metadata, clause)
+        fail!("condition must be boolean", clause) unless metadata[:type] == :boolean
+      end
+
+      def inspect_node(node, clause, visible)
+        fail!("unsupported expression", clause) unless node.is_a?(Node)
+        case node.kind
+        when :column
+          table, name = node.args
+          fail!("source #{table.label} is not visible", clause) unless visible.any? { |s| s.key == table.key }
+          definition = table.columns[name]
+          fail!("#{table.schema}.#{table.name}.#{name} is undeclared", clause) unless definition
+          fail!("#{table.schema}.#{table.name}.#{name} is disabled", clause) unless definition[:enabled]
+          dependency = { schema: table.schema, table: table.name, source: table.label, column: name, clause: clause }.freeze
+          @dependencies << dependency unless @dependencies.include?(dependency)
+          info(definition[:type], definition[:nullable] || @left_sources.include?(table.key))
+        when :as
+          fail!("projection aliases are only allowed in SELECT", clause) unless clause == "SELECT"
+          inspect_node(node.args[0], clause, visible)
+        when :value
+          value = node.args[0]
+          type = case value
+                 when NilClass then :null
+                 when Integer then :bigint
+                 when String then :text
+                 when TrueClass, FalseClass then :boolean
+                 when Time then :timestamptz
+                 else
+                   case value.class.to_s
+                   when "DateTime" then :timestamptz
+                   when "Date" then :date
+                   else fail!("unsupported bound value #{value.class}", clause)
+                   end
+                 end
+          info(type, value.nil?)
+        when :count
+          fail!("aggregate is not allowed", clause) if ["WHERE", "ON", "GROUP BY"].include?(clause)
+          fail!("COUNT DISTINCT requires a column", clause) if node.args.empty? && node.options[:distinct]
+          fail!("COUNT distinct flag must be boolean", clause) unless [true, false].include?(node.options[:distinct])
+          node.args.each do |arg|
+            fail!("nested aggregate", clause) if aggregate?(arg)
+            inspect_node(arg, clause, visible)
+          end
+          info(:bigint, false, true)
+        when :raw
+          type = node.options[:type]
+          fail!("unsupported raw type", clause) unless TYPES.key?(type)
+          [:nullable, :aggregate].each do |flag|
+            fail!("raw #{flag} must be boolean", clause) unless [true, false].include?(node.options[flag])
+          end
+          if node.options[:aggregate] && ["WHERE", "ON", "GROUP BY"].include?(clause)
+            fail!("aggregate is not allowed", clause)
+          end
+          deps = node.options[:dependencies]
+          fail!("raw dependencies must be an array of columns", clause) unless deps.is_a?(Array)
+          deps.each do |dep|
+            fail!("raw dependency must be a column", clause) unless dep.is_a?(Node) && dep.kind == :column
+            inspect_node(dep, clause, visible)
+          end
+          fail!("raw parts cannot be empty", clause) if node.args.empty?
+          node.args.each do |part|
+            unless part.is_a?(String) || (part.is_a?(Node) && part.kind == :value)
+              fail!("raw parts must be strings or SQL.bind nodes", clause)
+            end
+            inspect_node(part, clause, visible) if part.is_a?(Node)
+          end
+          info(type, node.options[:nullable], node.options[:aggregate])
+        when :eq, :not_eq, :gt, :gte, :lt, :lte
+          left = inspect_node(node.args[0], clause, visible)
+          right = inspect_node(node.args[1], clause, visible)
+          if right[:type] == :null && ![:eq, :not_eq].include?(node.kind)
+            fail!("NULL only supports eq and not_eq", clause)
+          end
+          compatible!(left, right, node.args[1], clause)
+          info(:boolean, right[:type] == :null ? false : left[:nullable] || right[:nullable])
+        when :ilike
+          left = inspect_node(node.args[0], clause, visible)
+          right = inspect_node(node.args[1], clause, visible)
+          fail!("ILIKE requires text operands", clause) unless left[:type] == :text && right[:type] == :text
+          info(:boolean, left[:nullable] || right[:nullable])
+        when :in, :not_in
+          left = inspect_node(node.args[0], clause, visible)
+          values = node.args[1]
+          fail!("membership requires an array", clause) unless values.is_a?(Array)
+          values.each do |value|
+            rhs = Node.new(:value, [value])
+            compatible!(left, inspect_node(rhs, clause, visible), rhs, clause)
+          end
+          info(:boolean, left[:nullable])
+        when :and, :or, :not
+          node.args.each { |arg| predicate!(inspect_node(arg, clause, visible), clause) }
+          info(:boolean, true)
+        else
+          fail!("unsupported expression #{node.kind}", clause)
+        end
+      end
+
+      def compatible!(left, right, rhs, clause)
+        return if right[:type] == :null || left[:type] == right[:type]
+        return if [:integer, :bigint].include?(left[:type]) && [:integer, :bigint].include?(right[:type])
+        # Bound text is passed with an unspecified PostgreSQL type, allowing e.g. an ID from a route.
+        return if rhs.kind == :value && right[:type] == :text
+        return if right[:type] == :date && left[:type] == :timestamptz
+        fail!("incompatible comparison types", clause)
+      end
+
+      def aggregate?(node)
+        return false unless node.is_a?(Node)
+        return true if node.kind == :count || (node.kind == :raw && node.options[:aggregate])
+        node.args.any? { |arg| arg.is_a?(Node) && aggregate?(arg) }
+      end
+
+      def check_group(node, clause)
+        return if state[:group].any? { |group| group.key == node.key }
+        return if node.kind == :count || node.kind == :value
+        return if node.kind == :raw && node.options[:aggregate]
+        fail!("nonaggregate expression must be explicitly grouped", clause) if [:column, :raw].include?(node.kind)
+        node.args.each { |arg| check_group(arg, clause) if arg.is_a?(Node) }
+      end
+    end
+  end
+end