rdbr 0.1.0

data/lib/rdbr/query/page.rb

@@ -0,0 +1,66 @@
+require 'base64'
+require 'date'
+
+module RDBr
+  module Query
+    Page = Data.define(:namespace, :relation, :columns, :rows, :page, :per_page, :has_next, :total_count) do
+      def self.build(**attributes)
+        new(
+          namespace: attributes.fetch(:namespace).to_s.freeze,
+          relation: attributes.fetch(:relation).to_s.freeze,
+          columns: attributes.fetch(:columns).map{|column| column.to_s.freeze }.freeze,
+          rows: normalize_rows(attributes.fetch(:rows)),
+          page: Integer(attributes.fetch(:page)),
+          per_page: Integer(attributes.fetch(:per_page)),
+          has_next: !!attributes.fetch(:has_next),
+          total_count: attributes[:total_count]&.then{|count| Integer(count) }
+        )
+      end
+
+      def total_pages
+        return unless total_count
+
+        [ (total_count.to_f / per_page).ceil, 1 ].max
+      end
+
+      def to_h
+        {
+          namespace: namespace,
+          relation: relation,
+          columns: columns,
+          rows: rows,
+          page: page,
+          per_page: per_page,
+          has_next: has_next,
+          total_count: total_count,
+          total_pages: total_pages
+        }
+      end
+
+      def self.normalize_rows(rows)
+        rows.map do |row|
+          row.to_h.transform_values{|value| normalize_value(value) }.freeze
+        end.freeze
+      end
+
+      def self.normalize_value(value)
+        case value
+        when Time, DateTime then value.iso8601(6)
+        when Date then value.iso8601
+        when String then normalize_string(value)
+        when Array then value.map{|item| normalize_value(item) }.freeze
+        when Hash then value.transform_values{|item| normalize_value(item) }.freeze
+        else value
+        end
+      end
+
+      def self.normalize_string(value)
+        return value.freeze if value.valid_encoding?
+
+        { 'encoding' => 'base64', 'data' => Base64.strict_encode64(value.b) }.freeze
+      end
+
+      private_class_method :normalize_rows, :normalize_value, :normalize_string
+    end
+  end
+end

data/lib/rdbr/query/postgresql.rb

@@ -0,0 +1,7 @@
+module RDBr
+  module Query
+    class Postgresql < Relational
+      include PostgresqlDialect
+    end
+  end
+end

data/lib/rdbr/query/postgresql_dialect.rb

@@ -0,0 +1,11 @@
+module RDBr
+  module Query
+    module PostgresqlDialect
+      private
+
+      def pattern_expression(identifier, value) = "#{identifier} ILIKE #{value}"
+
+      def text_expression(identifier) = "#{identifier}::text"
+    end
+  end
+end

data/lib/rdbr/query/relational.rb

@@ -0,0 +1,239 @@
+module RDBr
+  module Query
+    class Relational
+      include TypeSupport
+      include Lookup
+
+      Request = Data.define(:namespace, :relation, :page_number, :page_size, :sort, :filters, :search)
+      GroupRequest = Data.define(:namespace, :relation, :column, :page_number, :page_size, :filters, :search)
+      DEFAULT_PAGE_SIZE = 25
+
+      def initialize(connection:, catalog:, max_results: 100)
+        @connection = connection
+        @catalog = catalog
+        @max_results = positive_integer(max_results, 'maximum result limit')
+      end
+
+      def page(namespace:, relation:, **options)
+        namespace_metadata, relation_metadata = find_relation(namespace, relation)
+        request = Request.new(
+          namespace: namespace_metadata,
+          relation: relation_metadata,
+          page_number: positive_integer(options.fetch(:page, 1), 'page'),
+          page_size: [ positive_integer(options.fetch(:per_page, DEFAULT_PAGE_SIZE), 'page size'), max_results ].min,
+          sort: options[:sort],
+          filters: options.fetch(:filters, {}),
+          search: options[:search]
+        )
+        total_count = count_rows(request) if options[:include_total]
+        rows = connection.select_all(select_sql(request))
+        build_page(request, rows, total_count)
+      end
+
+      def build_page(request, rows, total_count)
+        Page.build(
+          namespace: request.namespace.name,
+          relation: request.relation.name,
+          columns: request.relation.columns.map(&:name),
+          rows: rows.first(request.page_size),
+          page: request.page_number,
+          per_page: request.page_size,
+          has_next: page_has_next?(request, rows, total_count),
+          total_count: total_count
+        )
+      end
+
+      def page_has_next?(request, rows, total_count)
+        return rows.length > request.page_size unless total_count
+
+        request.page_number * request.page_size < total_count
+      end
+
+      delegate :disconnect, to: :connection
+
+      def record(namespace:, relation:, key:)
+        _namespace, relation_metadata = find_relation(namespace, relation)
+        primary_key = relation_metadata.primary_key
+        raise InvalidQueryError, 'This relation has no primary key' unless primary_key
+
+        normalized_key = key.to_h.transform_keys(&:to_s)
+        unless normalized_key.keys.sort == primary_key.columns.sort
+          raise InvalidQueryError, 'Record key must contain every primary-key column'
+        end
+
+        page(namespace: namespace, relation: relation, per_page: 1, filters: normalized_key).rows.first
+      end
+
+      def group(namespace:, relation:, column:, **options)
+        namespace_metadata, relation_metadata = find_relation(namespace, relation)
+        column_metadata = relation_metadata.column(column)
+        raise InvalidQueryError, "Unknown group column: #{column}" unless column_metadata
+        raise InvalidQueryError, "Unsupported group column: #{column}" unless groupable_column?(column_metadata)
+
+        page_number = positive_integer(options.fetch(:page, 1), 'page')
+        requested_size = positive_integer(options.fetch(:per_page, DEFAULT_PAGE_SIZE), 'page size')
+        page_size = [ requested_size, max_results ].min
+        request = GroupRequest.new(
+          namespace: namespace_metadata, relation: relation_metadata, column: column_metadata,
+          page_number: page_number, page_size: page_size,
+          filters: options.fetch(:filters, {}), search: options[:search]
+        )
+        rows = connection.select_all(group_sql(request))
+        Page.build(
+          namespace: namespace_metadata.name, relation: relation_metadata.name,
+          columns: [ 'value', 'count' ], rows: rows.first(page_size), page: page_number,
+          per_page: page_size, has_next: rows.length > page_size
+        )
+      end
+
+      private
+
+      attr_reader :connection, :catalog, :max_results
+
+      def select_sql(request)
+        table = quoted_table(request)
+        clauses = [ "SELECT * FROM #{table}" ]
+        where = where_clause(request.relation, request.filters, request.search)
+        clauses << "WHERE #{where}" unless where.empty?
+        clauses << "ORDER BY #{order_clause(request.relation, request.sort)}"
+        clauses << pagination_clause(request)
+        clauses.join("\n")
+      end
+
+      def count_rows(request)
+        clauses = [ "SELECT COUNT(*) FROM #{quoted_table(request)}" ]
+        where = where_clause(request.relation, request.filters, request.search)
+        clauses << "WHERE #{where}" unless where.empty?
+        connection.select_value(clauses.join("\n")).to_i
+      end
+
+      def quoted_table(request)
+        [ request.namespace.name, request.relation.name ]
+          .map{|name| connection.quote_identifier(name) }.join('.')
+      end
+
+      def pagination_clause(request)
+        "LIMIT #{request.page_size + 1} OFFSET #{(request.page_number - 1) * request.page_size}"
+      end
+
+      def group_sql(request)
+        table = quoted_table(request)
+        identifier = connection.quote_identifier(request.column.name)
+        offset = (request.page_number - 1) * request.page_size
+        clauses = [ "SELECT #{identifier} AS value, COUNT(*) AS count", "FROM #{table}" ]
+        where = where_clause(request.relation, request.filters, request.search)
+        clauses << "WHERE #{where}" unless where.empty?
+        clauses << "GROUP BY #{identifier}"
+        clauses << "ORDER BY count DESC, #{text_expression(identifier)} ASC"
+        clauses << "LIMIT #{request.page_size + 1} OFFSET #{offset}"
+        clauses.join("\n")
+      end
+
+      def find_relation(namespace_name, relation_name)
+        namespace = catalog.namespace(namespace_name)
+        relation = namespace&.relation(relation_name)
+        raise RelationNotFoundError, 'Unknown namespace or relation' unless relation
+
+        [ namespace, relation ]
+      end
+
+      def where_clause(relation, filters, search = nil)
+        clauses = normalize_filters(filters).map do |column_name, operator, value|
+          column = relation.column(column_name)
+          raise InvalidQueryError, "Unknown filter column: #{column_name}" unless column
+
+          validate_operator!(column, operator)
+          filter_expression(column, operator, value)
+        end
+        clauses << search_expression(relation, search) unless search.to_s.empty?
+        clauses.join(' AND ')
+      end
+
+      def search_expression(relation, search)
+        columns = relation.columns.select{|column| type_group(column) == :text }
+        raise InvalidQueryError, 'This relation has no searchable text columns' if columns.empty?
+
+        value = connection.quote("%#{escape_pattern(search)}%")
+        expressions = columns.map{|column| pattern_expression(connection.quote_identifier(column.name), value) }
+        "(#{expressions.join(' OR ')})"
+      end
+
+      def normalize_filters(filters)
+        filters.to_h.flat_map do |column, condition|
+          if condition.is_a?(Hash)
+            condition.map{|operator, value| [ column.to_s, operator.to_sym, value ] }
+          else
+            [[ column.to_s, :eq, condition ]]
+          end
+        end
+      end
+
+      def validate_operator!(column, operator)
+        return if FILTER_OPERATORS.fetch(type_group(column)).include?(operator)
+
+        raise InvalidQueryError, "Unsupported filter operator #{operator} for #{column.name}"
+      end
+
+      def filter_expression(column, operator, value)
+        identifier = connection.quote_identifier(column.name)
+        return null_expression(identifier, value) if operator == :is_null
+        return "#{identifier} >= #{connection.quote(value)}" if operator == :from
+        return "#{identifier} < #{connection.quote(day_after(value))}" if operator == :to
+
+        quoted = connection.quote(filter_value(operator, value))
+        return pattern_expression(identifier, quoted) if [ :contains, :starts_with ].include?(operator)
+
+        sql_operator = { eq: '=', gt: '>', gte: '>=', lt: '<', lte: '<=' }
+        "#{identifier} #{sql_operator.fetch(operator)} #{quoted}"
+      end
+
+      def null_expression(identifier, value)
+        null = [ true, 'true', '1', 1 ].include?(value)
+        "#{identifier} IS #{null ? '' : 'NOT '}NULL"
+      end
+
+      def order_clause(relation, sort)
+        requested = parse_sort(relation, sort)
+        tie_breakers = deterministic_columns(relation).reject do |column|
+          requested.any?{|item| item.first == column.name }
+        end
+        parts = requested.map{|column, direction| "#{connection.quote_identifier(column.name)} #{direction}" }
+        parts.concat(tie_breakers.map{|column| deterministic_order(column) })
+        parts.empty? ? '(SELECT 0)' : parts.join(', ')
+      end
+
+      def deterministic_order(column)
+        identifier = connection.quote_identifier(column.name)
+        integer_column?(column) ? "#{identifier} ASC" : "#{text_expression(identifier)} ASC"
+      end
+
+      def parse_sort(relation, sort)
+        return [] if sort.nil? || sort.to_s.empty?
+
+        sort.to_s.split(',').map do |item|
+          descending = item.start_with?('-')
+          name = item.delete_prefix('-')
+          column = relation.column(name)
+          raise InvalidQueryError, "Unknown sort column: #{name}" unless column
+          raise InvalidQueryError, "Unsupported sort column: #{name}" unless sortable_column?(column)
+
+          [ column, descending ? 'DESC' : 'ASC' ]
+        end
+      end
+
+      def deterministic_columns(relation)
+        names = relation.primary_key&.columns || relation.columns.map(&:name)
+        names.filter_map{|name| relation.column(name) }
+      end
+
+      def positive_integer(value, name)
+        integer = Integer(value)
+        raise InvalidQueryError, "#{name} must be positive" unless integer.positive?
+
+        integer
+      rescue ArgumentError, TypeError
+        raise InvalidQueryError, "#{name} must be a positive integer"
+      end
+    end
+  end
+end

data/lib/rdbr/query/sqlite.rb

@@ -0,0 +1,15 @@
+module RDBr
+  module Query
+    class Sqlite < Relational
+      private
+
+      def pattern_expression(identifier, value)
+        "#{identifier} LIKE #{value} ESCAPE '\\'"
+      end
+
+      def text_expression(identifier)
+        "CAST(#{identifier} AS TEXT)"
+      end
+    end
+  end
+end

data/lib/rdbr/query/type_support.rb

@@ -0,0 +1,68 @@
+module RDBr
+  module Query
+    module TypeSupport
+      FILTER_OPERATORS = {
+        text: [ :eq, :contains, :starts_with, :is_null ],
+        numeric: [ :eq, :gt, :gte, :lt, :lte, :is_null ],
+        temporal: [ :eq, :gt, :gte, :lt, :lte, :from, :to, :is_null ],
+        identifier: [ :eq, :is_null ],
+        boolean: [ :eq, :is_null ],
+        other: [ :eq, :is_null ]
+      }.freeze
+      TYPE_GROUPS = {
+        text: %i[bpchar char citext name text varchar],
+        numeric: %i[decimal float4 float8 int2 int4 int8 money numeric oid],
+        temporal: %i[date interval time timestamp timestamptz timetz],
+        identifier: %i[uuid],
+        boolean: %i[bool boolean]
+      }.freeze
+      INTEGER_TYPES = %i[int2 int4 int8 oid].freeze
+
+      def filter_operators(column)
+        FILTER_OPERATORS.fetch(type_group(column))
+      end
+
+      def sortable_column?(column)
+        type_group(column) != :other
+      end
+
+      def groupable_column?(column)
+        !%i[json xml].include?(column.logical_type.to_s.to_sym)
+      end
+
+      private
+
+      def type_group(column)
+        logical_type = column.logical_type.to_s.to_sym
+        TYPE_GROUPS.find{|_group, types| types.include?(logical_type) }&.first || :other
+      end
+
+      def integer_column?(column)
+        INTEGER_TYPES.include?(column.logical_type.to_s.to_sym)
+      end
+
+      def day_after(value)
+        text = value.to_s
+        unless /\A\d{4}-\d{2}-\d{2}\z/.match?(text)
+          raise InvalidQueryError, 'The to filter requires a date in YYYY-MM-DD format'
+        end
+
+        (Date.iso8601(text) + 1).iso8601
+      rescue Date::Error
+        raise InvalidQueryError, 'The to filter requires a valid date in YYYY-MM-DD format'
+      end
+
+      def filter_value(operator, value)
+        escaped = escape_pattern(value)
+        return "%#{escaped}%" if operator == :contains
+        return "#{escaped}%" if operator == :starts_with
+
+        value
+      end
+
+      def escape_pattern(value)
+        value.to_s.gsub(/[\\%_]/){|character| "\\#{character}" }
+      end
+    end
+  end
+end

data/lib/rdbr/query/value_limiter.rb

@@ -0,0 +1,38 @@
+module RDBr
+  module Query
+    class ValueLimiter
+      SUFFIX = '... [value truncated]'.freeze
+
+      def initialize(max_bytes:)
+        @max_bytes = positive_integer(max_bytes)
+      end
+
+      def call(value)
+        case value
+        when String then truncate(value)
+        when Array then value.map{|item| call(item) }
+        when Hash then value.to_h{|key, item| [ key, call(item) ] }
+        else value
+        end
+      end
+
+      private
+
+      def truncate(value)
+        return value if value.bytesize <= @max_bytes
+
+        return SUFFIX.byteslice(0, @max_bytes) if @max_bytes <= SUFFIX.bytesize
+
+        available = @max_bytes - SUFFIX.bytesize
+        "#{value.byteslice(0, available).to_s.scrub}#{SUFFIX}"
+      end
+
+      def positive_integer(value)
+        number = Integer(value)
+        raise ArgumentError, 'maximum value size must be positive' unless number.positive?
+
+        number
+      end
+    end
+  end
+end

data/lib/rdbr/query.rb

@@ -0,0 +1,32 @@
+require_relative 'query/errors'
+require_relative 'query/page'
+require_relative 'query/value_limiter'
+require_relative 'query/connection'
+require_relative 'query/type_support'
+require_relative 'query/lookup'
+require_relative 'query/postgresql_dialect'
+require_relative 'query/relational'
+require_relative 'query/postgresql'
+require_relative 'query/sqlite'
+require_relative 'query/mysql'
+
+module RDBr
+  module Query
+    module_function
+
+    def build(database_url:, catalog:, max_results: 100, connection_options: {}, access_policy: nil)
+      adapter = catalog.adapter&.to_sym
+      query_class = { postgresql: Postgresql, sqlite: Sqlite, mysql: Mysql, mariadb: Mysql }[adapter]
+      raise UnsupportedAdapterError, "Data queries are not supported for #{catalog.adapter}" unless query_class
+
+      connection = Connection.new(
+        database_url,
+        options: connection_options,
+        access_policy: access_policy
+      )
+      query_class.new(connection: connection, catalog: catalog, max_results: max_results)
+    rescue ArgumentError, TypeError => e
+      raise ConfigurationError, e.message
+    end
+  end
+end

data/lib/rdbr/version.rb

@@ -0,0 +1,3 @@
+module RDBr
+  VERSION = '0.1.0'.freeze
+end