lowdata 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4c2e8629cffb84f30179cf393797a47d5cb05f7f03091a2b2cc122d8053593a4
+  data.tar.gz: 1468121aa2d7c393a167f890ad009da216191ed2da0d83371ce33e05f3cc4100
+SHA512:
+  metadata.gz: fabdfb8df5462fd1274a435bcc4a1230f58cafb17f0ebc6ec03a4652b81b86f61ef877d4237c121510325d0a72280a3de0afcf81490aea61facd1dc6336786c9
+  data.tar.gz: 9311a3e886512bf5eb297690db47fb78ff5f81c9d9c267ab1a5b9a289de94cc784909a4b8bf6104ab6841b56cd9c2e4be8f3a13f2bc14f296448389d3f03cc07

data/lib/col.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class LowData
+  # Columns to sort by with ^, and optionally which of the sorted rows to keep:
+  #   Posts[:title] ^ Col[:views, :title]          # ORDER BY views, title
+  #   Posts[:title] ^ -Col[:views]                 # ORDER BY views DESC
+  #   Posts[:title] ^ -Col[:views] + Col[:title]   # ORDER BY views DESC, title
+  #   Posts[:title] ^ -Col[:views] * 10            # ... LIMIT 10
+  #   Posts[:title] ^ -Col[:views][20...40]        # ... LIMIT 20 OFFSET 20
+  #   (Posts[:title] ^ -Col[:views] / 20)[2]       # ... LIMIT 20 OFFSET 40
+  # Ruby binds -, +, *, / and [] tighter than ^, so the rows travel with the sort keys to the expression.
+  class Col
+    Key = ::Data.define(:column, :descending)
+
+    attr_reader :keys, :rows
+
+    def self.[](*columns)
+      invalid = columns.reject { |column| column.is_a?(Symbol) }
+      raise ArgumentError, "Col[] takes column names, e.g. Col[:views], not #{invalid.map(&:inspect).join(', ')}" if invalid.any?
+      raise ArgumentError, 'Col[] takes at least one column, e.g. Col[:views]' if columns.empty?
+
+      new(keys: columns.map { |column| Key.new(column:, descending: false) })
+    end
+
+    def initialize(keys:, rows: nil)
+      @keys = keys
+      @rows = rows
+      freeze
+    end
+
+    def -@
+      with(keys: keys.map { |key| key.with(descending: !key.descending) })
+    end
+
+    def +(other)
+      raise ArgumentError, "Add sort columns with Col[], e.g. -Col[:views] + Col[:title], not #{other.inspect}" unless other.is_a?(Col)
+
+      with(keys: keys + other.keys, rows: Rows.merge(rows, other.rows))
+    end
+
+    def *(count)
+      with(rows: Rows.merge(rows, Rows.limit(count)))
+    end
+
+    def /(size)
+      with(rows: Rows.merge(rows, Rows.pages(size)))
+    end
+
+    def [](range)
+      with(rows: Rows.merge(rows, Rows.range(range)))
+    end
+
+    private
+
+    def with(keys: @keys, rows: @rows)
+      self.class.new(keys:, rows:)
+    end
+  end
+end

data/lib/data_expression.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+class LowData
+  # Joins tables with operators and builds a join tree for QueryCompiler:
+  #   Users[:username, Posts[:title]]   # Nested; every user once, with their posts nested. See Nesting.
+  #   Users[:username] + Posts[:title]   # Nested, shorthand for the above.
+  #   Users[:username] & Posts[:title]   # Inner join.
+  #   Users[:username] < Posts[:title]   # Left join.
+  #   Users[:username] > Posts[:title]   # Right join.
+  #   Users[:username] <=> Posts[:title] # Full outer join.
+  #   Table[:username, :title]           # Inverted; the repository's table joined to whichever tables own the columns.
+  # And sorts and limits the result. See Col and Rows:
+  #   Posts[:title] ^ -Col[:views] * 10  # ORDER BY views DESC LIMIT 10
+  #   (Posts[:title] / 20)[2]            # LIMIT 20 OFFSET 40
+  class DataExpression < ::Expressions::Expression
+    # The filters in a table's round brackets: `Posts(published: true) { views > 100 }`. See FilterCompiler.
+    Filter = ::Data.define(:values, :conditions, :related) do
+      def initialize(values: {}, conditions: [], related: [])
+        super
+      end
+
+      def merge(other)
+        with(values: values.merge(other.values), conditions: conditions + other.conditions, related: related + other.related)
+      end
+    end
+    # A table in another table's round brackets, keeping rows with `count` related rows: `Users(Posts => 3..)`.
+    Related = ::Data.define(:key, :source, :count)
+    # A table with the columns to select, the tables nested in its brackets and the filter on its rows.
+    Source = ::Data.define(:table, :columns, :nested, :filter) do
+      def initialize(table:, columns:, nested: [], filter: Filter.new)
+        super
+      end
+    end
+    # A table nested in another table's brackets, under an optional key: `Reviews[:rating, author: Users[:username]]`.
+    Nested = ::Data.define(:key, :node)
+    Join = ::Data.define(:type, :left, :right)
+    Inversion = ::Data.define(:table, :columns, :filter)
+    # A node sorted by `keys` (Col::Key) with `rows` (Rows) kept: `Posts[:title] ^ -Col[:views] * 10`.
+    Ordered = ::Data.define(:node, :keys, :rows)
+
+    JOIN_HOOKS = { add: :nest, intersect: :inner, lt: :left, gt: :right, compare: :full }.freeze
+
+    attr_reader :node
+
+    def initialize(node:)
+      @node = node
+    end
+
+    SORT_MESSAGE = 'Sort by columns with Col[], e.g. Posts[:title] ^ Col[:views], not %<value>s'
+    NESTED_SQL_MESSAGE = '+ loads nested data with one query per table, so it has no single SQL query. Call to_a instead'
+
+    def dataset(database: LowData.database)
+      raise Error, NESTED_SQL_MESSAGE if nested?
+
+      QueryCompiler.new(database:).compile(node: @node)
+    end
+
+    def sql(database: LowData.database)
+      dataset(database:).sql
+    end
+
+    def to_a(database: LowData.database)
+      return Nesting.new(database:).load(node: @node) if nested?
+
+      dataset(database:).all
+    end
+
+    # Picks a page after splitting rows into pages: `(Posts[:title] / 20)[2]` is the third page of 20. Returns a new
+    # expression, so the same pages can be picked from more than once.
+    def [](page)
+      ordered = @node if @node.is_a?(Ordered)
+      raise ArgumentError, 'Split rows into pages before picking one, e.g. (Posts[:title] / 20)[0]' unless ordered&.rows
+
+      DataExpression.new(node: ordered.with(rows: ordered.rows.page(page)))
+    end
+
+    private
+
+    def nested?
+      node = @node.is_a?(Ordered) ? @node.node : @node
+      (node.is_a?(Join) && node.type == :nest) || (node.is_a?(Source) && node.nested.any?)
+    end
+
+    # Sorting again adds sort keys after the existing ones, and an expression keeps one limit, range or page size.
+    def order(keys:, rows:)
+      ordered = @node.is_a?(Ordered) ? @node : Ordered.new(node: @node, keys: [], rows: nil)
+      @node = ordered.with(keys: ordered.keys + keys, rows: Rows.merge(ordered.rows, rows))
+    end
+
+    def xor_value(value)
+      raise ArgumentError, format(SORT_MESSAGE, value: value.inspect) unless value.is_a?(Col)
+
+      order(keys: value.keys, rows: value.rows)
+    end
+
+    def multiply_value(count)
+      order(keys: [], rows: Rows.limit(count))
+    end
+
+    def divide_value(size)
+      order(keys: [], rows: Rows.pages(size))
+    end
+
+    def xor_expression(_expression)
+      raise ArgumentError, format(SORT_MESSAGE, value: 'a data expression')
+    end
+
+    def xor_type(type_)
+      raise ArgumentError, format(SORT_MESSAGE, value: type_)
+    end
+
+    { multiply: 'Limit', divide: 'Split into pages' }.each do |prefix, action|
+      define_method(:"#{prefix}_expression") do |_expression|
+        raise ArgumentError, "#{action} by a positive integer, not a data expression"
+      end
+
+      define_method(:"#{prefix}_type") do |type_|
+        raise ArgumentError, "#{action} by a positive integer, not the type #{type_}"
+      end
+    end
+
+    JOIN_HOOKS.each do |prefix, type|
+      define_method(:"#{prefix}_expression") do |expression|
+        @node = Join.new(type:, left: @node, right: expression.node)
+      end
+
+      define_method(:"#{prefix}_value") do |value|
+        raise ArgumentError, "Join a data expression such as Posts[:title], not the value #{value.inspect}"
+      end
+
+      define_method(:"#{prefix}_type") do |type_|
+        raise ArgumentError, "Join a data expression such as Posts[:title], not the type #{type_}"
+      end
+    end
+  end
+end

data/lib/errors.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class LowData
+  class Error < StandardError; end
+
+  class ConfigError < Error
+    MISSING_DATABASE = 'Set LowData.database to a Sequel database, e.g. LowData.database = Sequel.sqlite'
+  end
+
+  class JoinError < Error; end
+end

data/lib/filter_compiler.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class LowData
+  # Compiles a table's filter into WHERE conditions, each qualified with the table:
+  #   Posts(published: true)        # WHERE posts.published IS TRUE
+  #   Posts { views > 100 }         # WHERE posts.views > 100
+  #   Users(Posts)                  # WHERE EXISTS (SELECT 1 FROM posts WHERE posts.user_id = users.id)
+  #   Users(Posts => 3..)           # WHERE (SELECT count(*) FROM posts WHERE posts.user_id = users.id) >= 3
+  class FilterCompiler
+    def initialize(database:, schema:)
+      @database = database
+      @links = NestingTree.new(schema:)
+    end
+
+    def apply(dataset:, source:)
+      conditions(source:).reduce(dataset) { |filtered, condition| filtered.where(condition) }
+    end
+
+    private
+
+    def conditions(source:)
+      table = source.table
+      filter = source.filter
+      values = filter.values.map { |column, value| { Sequel[table][column] => value } }
+      conditions = filter.conditions.map { |condition| Sequel::Qualifier.new(table).transform(condition) }
+
+      values + conditions + filter.related.map { |related| related_condition(table:, related:) }
+    end
+
+    # Correlates the related table's rows with the outer table's by the foreign key that links them, as nesting does.
+    def related_condition(table:, related:)
+      inner = related.source.table
+      raise JoinError, "Can't filter #{table} by its own rows" if inner == table
+
+      _key, link = @links.link_to(parent: table, child: inner, key: related.key)
+      rows = @database[inner].where(Sequel[inner][link.child_column] => Sequel[table][link.parent_column])
+      count(rows: apply(dataset: rows, source: related.source), count: related.count)
+    end
+
+    def count(rows:, count:)
+      return rows.select(1).exists if count == (1..)
+      return ~rows.select(1).exists if [0, 0..0].include?(count)
+
+      { rows.select(Sequel.function(:count).*) => count }
+    end
+  end
+end

data/lib/inversion.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class LowData
+  # Resolves `Table[:username, :title]` in PostsData to a join tree: posts left joined to each related table that owns a column.
+  class Inversion
+    def initialize(schema:)
+      @schema = schema
+    end
+
+    # Returns the join tree and the columns to select, in the order they were asked for.
+    def resolve(node:)
+      owners = node.columns.to_h { |column| [column, owner(table: node.table, column:)] }
+      base = DataExpression::Source.new(table: node.table, columns: [], filter: node.filter)
+
+      tree = (owners.values.uniq - [node.table]).reduce(base) do |left, table|
+        DataExpression::Join.new(type: :left, left:, right: DataExpression::Source.new(table:, columns: []))
+      end
+
+      [tree, owners.map { |column, table| Sequel[table][column] }]
+    end
+
+    private
+
+    def owner(table:, column:)
+      return table if @schema.columns(table:).include?(column)
+
+      owners = @schema.related_tables(table:).select { |related| @schema.columns(table: related).include?(column) }
+      raise JoinError, "No table related to #{table} has a #{column} column" if owners.empty?
+      raise JoinError, ambiguous_message(table:, column:, owners:) if owners.length > 1
+
+      owners.first
+    end
+
+    def ambiguous_message(table:, column:, owners:)
+      "#{owners.join(' and ')} both have a #{column} column and are related to #{table}. " \
+        "Join explicitly instead, e.g. #{LowData.constant_name(table)}[] < #{LowData.constant_name(owners.first)}[:#{column}]"
+    end
+  end
+end

data/lib/lowdata.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'expressions'
+require 'lowtype'
+require 'sequel'
+
+require_relative 'version'
+require_relative 'errors'
+require_relative 'data_expression'
+require_relative 'rows'
+require_relative 'col'
+require_relative 'order_compiler'
+require_relative 'filter_compiler'
+require_relative 'schema'
+require_relative 'inversion'
+require_relative 'nesting'
+require_relative 'nesting_tree'
+require_relative 'query_compiler'
+require_relative 'table_reference'
+
+class LowData
+  class << self
+    attr_reader :database
+    attr_writer :table
+
+    def database=(database)
+      @database = database
+      @tables = nil
+    end
+
+    # The database's tables, cached until the database changes.
+    def tables
+      @tables ||= database.tables
+    end
+
+    # The repository's table, named after the class: PostsData is :posts and BlogPostsData is :blog_posts.
+    def table
+      @table ||= name && table_name(name.split('::').last.delete_suffix('Data'))
+    end
+
+    # Resolve undefined constants in a repository to tables, so `Users` and `BlogPosts` become :users and :blog_posts.
+    # `Table` is the repository's own table, joined to whichever related tables own the columns asked for.
+    def const_missing(name)
+      return super if self == LowData
+      return inverted_table if name == :Table
+
+      TableReference.new(table: table_name(name))
+    end
+
+    # The constant a table is written as in a data expression: :blog_posts is BlogPosts.
+    def constant_name(table)
+      table.to_s.split('_').map(&:capitalize).join
+    end
+
+    def table_name(constant_name)
+      constant_name.to_s.gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase.to_sym
+    end
+
+    private
+
+    def inverted_table
+      raise ConfigError, "Set the table for #{self}, e.g. self.table = :posts" if table.nil?
+
+      TableReference.new(table:, inverted: true)
+    end
+  end
+
+  # Round brackets after a table filter it: `Posts(published: true) { views > 100 }[:title]`. A capitalised name is a
+  # table only when the database has that table, so typos still raise NoMethodError. `Table(...)` is always a table.
+  def method_missing(name, *arguments, **keywords, &block)
+    raise ConfigError, ConfigError::MISSING_DATABASE if LowData.database.nil? && name.match?(/\A[A-Z]/)
+    return super unless table_method?(name:)
+
+    self.class.const_missing(name).where(*arguments, **keywords, &block)
+  end
+
+  def respond_to_missing?(name, include_private = false)
+    table_method?(name:) || super
+  end
+
+  private
+
+  def table_method?(name:)
+    return false unless name.match?(/\A[A-Z]/)
+    return true if name == :Table
+    return false if LowData.database.nil?
+
+    LowData.tables.include?(LowData.table_name(name))
+  end
+end

data/lib/nesting.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+class LowData
+  # Loads nested data, one query per table. Tables nest explicitly in brackets, or by inference with +:
+  #   Posts[:title, Users[:username]]  # => [{ title: 'Hello', user: { username: 'alice' } }]
+  #   Users[:username] + Posts[:title] # => [{ username: 'alice', posts: [{ title: 'Hello' }] }]
+  # In a + chain the first table is the subject, and each other operand nests under the table its foreign key links it to.
+  class Nesting
+    # A table in the nesting tree. `link` connects it to its parent node: `parent_column` on the parent's rows matches
+    # `child_column` on this table's rows, and `many` nests the matches as an array rather than one object.
+    # `keys` and `rows` sort and limit the table's rows, per parent when it's nested.
+    Node = ::Data.define(:source, :key, :link, :children, :keys, :rows) do
+      def initialize(source:, key:, link:, children:, keys: [], rows: nil)
+        super
+      end
+    end
+    Link = ::Data.define(:parent_column, :child_column, :many)
+
+    ROW_NUMBER = :lowdata_row_number
+
+    def initialize(database:)
+      raise ConfigError, ConfigError::MISSING_DATABASE if database.nil?
+
+      @database = database
+      @schema = Schema.new(database:)
+      @filters = FilterCompiler.new(database:, schema: @schema)
+      @orders = OrderCompiler.new(schema: @schema)
+    end
+
+    def load(node:)
+      root = NestingTree.new(schema: @schema).build(node:)
+      shape(node: root, rows: fetch(node: root))
+    end
+
+    private
+
+    def fetch(node:, filter: nil)
+      dataset = @filters.apply(dataset: @database[node.source.table].select(*select_columns(node:)), source: node.source)
+      dataset = dataset.where(filter) if filter
+      return per_parent(node:, dataset:) if filter && node.rows
+
+      @orders.apply(dataset:, keys: node.keys, rows: node.rows, tables: [node.source.table]).all
+    end
+
+    # Keeps each parent's rows, e.g. each user's 3 most viewed posts, by numbering the rows within each parent.
+    def per_parent(node:, dataset:)
+      numbered = dataset.select_append(row_number(node:).as(ROW_NUMBER))
+      @database.from(numbered).where(ROW_NUMBER => row_numbers(rows: node.rows)).order(ROW_NUMBER)
+               .all.map { |row| row.except(ROW_NUMBER) }
+    end
+
+    def row_number(node:)
+      table = node.source.table
+      order = @orders.order(keys: node.keys, tables: [table])
+      Sequel.function(:row_number).over(partition: Sequel[table][node.link.child_column], order: order.empty? ? nil : order)
+    end
+
+    # Row numbers start at 1, and rows count from 0.
+    def row_numbers(rows:)
+      @orders.check_rows!(rows:)
+      first = rows.offset + 1
+      rows.limit ? first..(rows.offset + rows.limit) : (first..)
+    end
+
+    # The columns asked for, plus the columns each link needs.
+    def select_columns(node:)
+      return [Sequel[node.source.table].*] if node.source.columns.empty?
+
+      link_columns = [node.link&.child_column, *node.children.map { |child| child.link.parent_column }].compact
+      (node.source.columns + link_columns).uniq
+    end
+
+    # Turns fetched rows into output hashes: the columns asked for, plus each child nested under its key.
+    def shape(node:, rows:)
+      nested = node.children.to_h { |child| [child, nested_rows(child:, rows:)] }
+
+      rows.map do |row|
+        output = node.source.columns.empty? ? row.dup : row.slice(*node.source.columns)
+        node.children.each { |child| output[child.key] = matches(child:, row:, nested: nested[child]) }
+        output
+      end
+    end
+
+    def matches(child:, row:, nested:)
+      matches = nested.fetch(row[child.link.parent_column], [])
+      child.link.many ? matches : matches.first
+    end
+
+    # Fetches a child's rows for all parent rows in one query, grouped by the value that links them.
+    def nested_rows(child:, rows:)
+      values = rows.map { |row| row[child.link.parent_column] }.compact.uniq
+      return {} if values.empty?
+
+      child_rows = fetch(node: child, filter: { child.link.child_column => values })
+      child_rows.zip(shape(node: child, rows: child_rows))
+                .group_by { |row, _output| row[child.link.child_column] }
+                .transform_values { |pairs| pairs.map(&:last) }
+    end
+  end
+end

data/lib/nesting_tree.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+class LowData
+  # Builds the tree Nesting loads from an expression. Tables in brackets nest directly under their table, and the
+  # operands of a + chain nest by following foreign keys from the first operand.
+  class NestingTree
+    MULTIPLE_PATHS_MESSAGE = 'Foreign keys link %<tables>s by more than one path, so + has no single way to nest them. ' \
+                             'Nest them in brackets instead, e.g. Users[:username, Posts[:title, Likes[]]]'
+
+    def initialize(schema:)
+      @schema = schema
+    end
+
+    # Builds the tree for an expression: a table with any tables nested in its brackets, or a + chain.
+    def build(node:)
+      raise JoinError, QueryCompiler::INVERSION_JOIN_MESSAGE if node.is_a?(DataExpression::Inversion)
+      return build(node: node.node).with(keys: node.keys, rows: node.rows) if node.is_a?(DataExpression::Ordered)
+      return bracketed(source: node) if node.is_a?(DataExpression::Source)
+
+      chain(operands: operands(node:).map { |operand| build(node: operand) })
+    end
+
+    # Returns the nested key and Link for a child table. On the "one" side a key matching a foreign key column picks it,
+    # so `author:` follows `author_id`. FilterCompiler links related tables the same way.
+    def link_to(parent:, child:, key:)
+      to_one = @schema.links(from: parent, to: child)
+      named = named_link(links: to_one, key:)
+      return one(parent:, child:, link: named, key:) if named
+
+      links = to_one + @schema.links(from: child, to: parent)
+      raise JoinError, "No foreign key links #{child} to #{parent}" if links.empty?
+      raise JoinError, many_keys_message(parent:, child:, to_one:) if links.length > 1
+      return one(parent:, child:, link: links.first, key:) if to_one.any?
+
+      many(parent:, child:, link: links.first, key:)
+    end
+
+    private
+
+    # A + chain is left deep, and each operand is a table, a bracketed table or a parenthesised + chain.
+    def operands(node:)
+      return [node] unless node.is_a?(DataExpression::Join)
+      raise JoinError, QueryCompiler::NESTING_JOIN_MESSAGE unless node.type == :nest
+
+      operands(node: node.left) + [node.right]
+    end
+
+    # Tables in brackets nest directly under the table whose brackets they're in.
+    def bracketed(source:)
+      children = source.nested.map do |nested|
+        child = build(node: nested.node)
+        key, link = link_to(parent: source.table, child: child.source.table, key: nested.key)
+        child.with(key:, link:)
+      end
+
+      with_children(node: Nesting::Node.new(source:, key: nil, link: nil, children: []), children:)
+    end
+
+    # Links the operands into a tree from the first. Every operand must be linked, and only by one path.
+    def chain(operands:)
+      check_duplicates!(operands:)
+
+      edges = operands.combination(2).select { |operand, other| linked?(operand.source.table, other.source.table) }
+      visited = [operands.first]
+      root = attach(operand: operands.first, node: operands.first, edges:, visited:)
+      check_chain!(operands:, visited:, edges:)
+
+      root
+    end
+
+    # Nests each unvisited operand linked to `operand`. `node` is the operand with its key and link set.
+    def attach(operand:, node:, edges:, visited:)
+      neighbours = edges.filter_map { |pair| (pair - [operand]).first if pair.include?(operand) } - visited
+      visited.concat(neighbours)
+
+      children = neighbours.map do |child|
+        key, link = link_to(parent: node.source.table, child: child.source.table, key: nil)
+        attach(operand: child, node: child.with(key:, link:), edges:, visited:)
+      end
+
+      with_children(node:, children:)
+    end
+
+    def with_children(node:, children:)
+      children = node.children + children
+      check_keys!(table: node.source.table, children:)
+      node.with(children:)
+    end
+
+    def check_duplicates!(operands:)
+      tables = operands.map { |operand| operand.source.table }
+      duplicate = tables.find { |table| tables.count(table) > 1 }
+      raise JoinError, "Can't join #{duplicate} to itself. Name each in brackets, e.g. author: Users[:username]" if duplicate
+    end
+
+    def check_chain!(operands:, visited:, edges:)
+      tables = operands.map { |operand| operand.source.table }
+      unreached = (operands - visited).map { |operand| operand.source.table }
+      raise JoinError, "No foreign key links #{unreached.join(' and ')} to #{(tables - unreached).join(', ')}" if unreached.any?
+      raise JoinError, format(MULTIPLE_PATHS_MESSAGE, tables: tables.join(', ')) if edges.length > operands.length - 1
+    end
+
+    def check_keys!(table:, children:)
+      keys = children.map(&:key)
+      duplicate = keys.find { |key| keys.count(key) > 1 }
+      raise JoinError, "Two tables nest in #{table} as #{duplicate}. Name them, e.g. author: Users[:username]" if duplicate
+    end
+
+    def linked?(table, other)
+      @schema.links(from: table, to: other).any? || @schema.links(from: other, to: table).any?
+    end
+
+    def named_link(links:, key:)
+      named = links.select { |link| key && [key, :"#{key}_id"].include?(link[:columns].first) }
+      named.first if named.one?
+    end
+
+    # `user_id` nests as `user:`, like the association name in most ORMs.
+    def one(parent:, child:, link:, key:)
+      column, foreign_key = sole_columns(parent:, child:, link:)
+      key ||= column.to_s.end_with?('_id') ? column.to_s.delete_suffix('_id').to_sym : child
+      [key, Nesting::Link.new(parent_column: column, child_column: foreign_key, many: false)]
+    end
+
+    def many(parent:, child:, link:, key:)
+      column, foreign_key = sole_columns(parent:, child:, link:)
+      [key || child, Nesting::Link.new(parent_column: foreign_key, child_column: column, many: true)]
+    end
+
+    def sole_columns(parent:, child:, link:)
+      columns, keys = link.values_at(:columns, :keys)
+      raise JoinError, "Nesting doesn't support composite foreign keys between #{parent} and #{child}" if columns.length > 1
+
+      [columns.first, keys.first]
+    end
+
+    def many_keys_message(parent:, child:, to_one:)
+      message = "More than one foreign key links #{parent} and #{child}"
+      return message if to_one.empty?
+
+      names = to_one.map { |link| "#{link[:columns].first.to_s.delete_suffix('_id')}: #{LowData.constant_name(child)}[]" }
+      "#{message}. Name the one to follow, e.g. #{names.sort.join(' or ')}"
+    end
+  end
+end

data/lib/order_compiler.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class LowData
+  # Compiles sort keys into ORDER BY expressions and rows into LIMIT and OFFSET. Each column is qualified with the one
+  # table that has it, so `Users[:username] & Posts[:title] ^ Col[:views]` sorts by posts.views.
+  class OrderCompiler
+    UNPICKED_PAGE_MESSAGE = 'Pick a page after splitting rows into pages, e.g. (Posts[:title] / 20)[0]'
+
+    def initialize(schema:)
+      @schema = schema
+    end
+
+    def apply(dataset:, keys:, rows:, tables:)
+      dataset = dataset.order(*order(keys:, tables:)) if keys.any?
+      return dataset unless rows
+
+      check_rows!(rows:)
+      dataset.limit(rows.limit, rows.offset.zero? ? nil : rows.offset)
+    end
+
+    def order(keys:, tables:)
+      keys.map do |key|
+        column = Sequel[owner(column: key.column, tables:)][key.column]
+        key.descending ? column.desc : column
+      end
+    end
+
+    def check_rows!(rows:)
+      raise ArgumentError, UNPICKED_PAGE_MESSAGE if rows.page_size
+    end
+
+    private
+
+    def owner(column:, tables:)
+      owners = tables.select { |table| @schema.columns(table:).include?(column) }
+      raise JoinError, "No table in #{tables.join(', ')} has a #{column} column to sort by" if owners.empty?
+      raise JoinError, "#{owners.join(' and ')} both have a #{column} column, so Col[:#{column}] is ambiguous" if owners.length > 1
+
+      owners.first
+    end
+  end
+end

data/lib/query_compiler.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+class LowData
+  # Compiles a DataExpression join tree into a Sequel dataset, inferring each join's ON clause from foreign keys.
+  class QueryCompiler
+    INVERSION_JOIN_MESSAGE = "Table[] can't be joined with other expressions"
+    NESTING_JOIN_MESSAGE = 'Nested tables return nested data, so + and tables nested in brackets can\'t be combined with &, <, > or <=>'
+    ORDERED_JOIN_MESSAGE = 'Sort and limit after joining, e.g. (Users[] < Posts[]) ^ Col[:username]. Ruby binds ^ tighter ' \
+                           'than <, > and <=>, and * and / tighter than every join, so add parentheses around the join'
+
+    def initialize(database:)
+      raise ConfigError, ConfigError::MISSING_DATABASE if database.nil?
+
+      @database = database
+      @schema = Schema.new(database:)
+      @filters = FilterCompiler.new(database:, schema: @schema)
+      @orders = OrderCompiler.new(schema: @schema)
+    end
+
+    def compile(node:)
+      return compile_ordered(node:) if node.is_a?(DataExpression::Ordered)
+
+      compile_unordered(node:, sources: [])
+    end
+
+    private
+
+    # Sort keys and rows apply to the whole query, so each sort column can belong to any joined table.
+    def compile_ordered(node:)
+      sources = []
+      dataset = compile_unordered(node: node.node, sources:)
+      @orders.apply(dataset:, keys: node.keys, rows: node.rows, tables: sources.map(&:table))
+    end
+
+    def compile_unordered(node:, sources:)
+      return compile_inversion(node:, sources:) if node.is_a?(DataExpression::Inversion)
+
+      dataset = filter(dataset: build(node:, sources:), sources:)
+      dataset.select(*sources.flat_map { |source| selections(source:) })
+    end
+
+    def compile_inversion(node:, sources:)
+      tree, selections = Inversion.new(schema: @schema).resolve(node:)
+      filter(dataset: build(node: tree, sources:), sources:).select(*selections)
+    end
+
+    def build(node:, sources:)
+      raise JoinError, INVERSION_JOIN_MESSAGE if node.is_a?(DataExpression::Inversion)
+      raise JoinError, ORDERED_JOIN_MESSAGE if node.is_a?(DataExpression::Ordered)
+      return source_dataset(source: node, sources:) if node.is_a?(DataExpression::Source)
+      raise JoinError, NESTING_JOIN_MESSAGE if nesting?(node:) || nesting?(node: node.right)
+
+      raise JoinError, nested_join_message if node.right.is_a?(DataExpression::Join)
+
+      join_dataset(dataset: build(node: node.left, sources:), type: node.type, right: node.right, sources:)
+    end
+
+    def join_dataset(dataset:, type:, right:, sources:)
+      raise JoinError, INVERSION_JOIN_MESSAGE if right.is_a?(DataExpression::Inversion)
+      raise JoinError, ORDERED_JOIN_MESSAGE if right.is_a?(DataExpression::Ordered)
+      raise JoinError, NESTING_JOIN_MESSAGE if right.nested.any?
+      raise JoinError, "Can't join #{right.table} to itself" if sources.any? { |source| source.table == right.table }
+
+      conditions = join_conditions(left_tables: sources.map(&:table), right_table: right.table)
+      sources << right
+      dataset.join_table(type, right.table, conditions)
+    end
+
+    def nesting?(node:)
+      node.is_a?(DataExpression::Join) && node.type == :nest
+    end
+
+    def source_dataset(source:, sources:)
+      raise JoinError, NESTING_JOIN_MESSAGE if source.nested.any?
+
+      sources << source
+      @database[source.table]
+    end
+
+    # Every table's filters go in the WHERE clause, so they apply to the joined rows.
+    def filter(dataset:, sources:)
+      sources.reduce(dataset) { |filtered, source| @filters.apply(dataset: filtered, source:) }
+    end
+
+    def selections(source:)
+      return [Sequel[source.table].*] if source.columns.empty?
+
+      source.columns.map { |column| Sequel[source.table][column] }
+    end
+
+    def join_conditions(left_tables:, right_table:)
+      candidates = left_tables.flat_map do |left_table|
+        @schema.references(from: right_table, to: left_table) + @schema.references(from: left_table, to: right_table)
+      end
+
+      raise JoinError, "No foreign key links #{right_table} to #{left_tables.join(', ')}" if candidates.empty?
+      raise JoinError, "More than one foreign key links #{right_table} to #{left_tables.join(', ')}" if candidates.length > 1
+
+      candidates.first
+    end
+
+    def nested_join_message
+      'Ruby evaluated a join on the right first, e.g. `Users[] < Posts[] & Comments[]` runs `Posts[] & Comments[]` first ' \
+        'because & binds tighter than <. Add parentheses to join from the left: `(Users[] < Posts[]) & Comments[]`'
+    end
+  end
+end

data/lib/rows.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+class LowData
+  # Which sorted rows to keep. Rows count from 0, as in Ruby:
+  #   Posts[] * 10                 # LIMIT 10
+  #   (Posts[] / 20)[2]            # LIMIT 20 OFFSET 40, the third page of 20
+  #   Posts[] ^ Col[:id][20...40]  # LIMIT 20 OFFSET 20
+  # Pages have a `page_size` until a page is picked, and a `limit` and `offset` after.
+  Rows = ::Data.define(:offset, :limit, :page_size) do
+    def self.limit(count)
+      new(offset: 0, limit: positive!(count:, operator: '*'), page_size: nil)
+    end
+
+    def self.pages(size)
+      new(offset: 0, limit: nil, page_size: positive!(count: size, operator: '/'))
+    end
+
+    def self.range(range)
+      check_range!(range:)
+      first = range.begin || 0
+      return new(offset: first, limit: nil, page_size: nil) if range.end.nil?
+
+      new(offset: first, limit: [range.end - first + (range.exclude_end? ? 0 : 1), 0].max, page_size: nil)
+    end
+
+    # One limit, range or page size per expression.
+    def self.merge(rows, other)
+      raise ArgumentError, 'An expression takes one limit, range or page size, not two' if rows && other
+
+      rows || other
+    end
+
+    def self.check_range!(range:)
+      unless range.is_a?(Range) && [range.begin, range.end].all? { |bound| bound.nil? || bound.is_a?(Integer) }
+        raise ArgumentError, "Pick rows with a range of integers, e.g. Col[:views][20...40], not #{range.inspect}"
+      end
+      raise ArgumentError, "Rows count from 0, so #{range.inspect} can't start below 0" if range.begin&.negative?
+    end
+
+    def self.positive!(count:, operator:)
+      return count if count.is_a?(Integer) && count.positive?
+
+      raise ArgumentError, "#{operator} takes a positive integer, e.g. Posts[:title] #{operator} 10, not #{count.inspect}"
+    end
+
+    def page(number)
+      raise ArgumentError, 'Split rows into pages before picking one, e.g. (Posts[:title] / 20)[0]' unless page_size
+      unless number.is_a?(Integer) && !number.negative?
+        raise ArgumentError, "Pages count from 0, so pick one with an integer from 0, not #{number.inspect}"
+      end
+
+      with(offset: number * page_size, limit: page_size, page_size: nil)
+    end
+  end
+end

data/lib/schema.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class LowData
+  # Reads tables, columns and foreign keys from the database, caching each lookup.
+  class Schema
+    def initialize(database:)
+      @database = database
+      @schemas = {}
+      @foreign_keys = {}
+    end
+
+    def columns(table:)
+      schema(table:).map(&:first)
+    end
+
+    def primary_key(table:)
+      schema(table:).select { |_column, info| info[:primary_key] }.map(&:first)
+    end
+
+    def foreign_keys(table:)
+      @foreign_keys[table] ||= @database.foreign_key_list(table)
+    end
+
+    # Tables with a foreign key to or from the table.
+    def related_tables(table:)
+      outgoing = foreign_keys(table:).map { |foreign_key| foreign_key[:table] }
+      incoming = @database.tables.select do |other|
+        foreign_keys(table: other).any? { |foreign_key| foreign_key[:table] == table }
+      end
+
+      (outgoing + incoming).uniq - [table]
+    end
+
+    # Foreign keys on the `from` table that reference the `to` table, as join conditions.
+    def references(from:, to:)
+      links(from:, to:).map do |link|
+        link[:columns].zip(link[:keys]).to_h { |column, key| [Sequel[from][column], Sequel[to][key]] }
+      end
+    end
+
+    # Foreign keys on the `from` table that reference the `to` table, as their columns and the keys they reference.
+    def links(from:, to:)
+      foreign_keys(table: from).select { |foreign_key| foreign_key[:table] == to }.map do |foreign_key|
+        { columns: foreign_key[:columns], keys: foreign_key[:key] || primary_key(table: to) }
+      end
+    end
+
+    private
+
+    def schema(table:)
+      @schemas[table] ||= @database.schema(table)
+    end
+  end
+end

data/lib/table_reference.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+class LowData
+  # A table named by a constant, such as `Users`, waiting for its columns: `Users[:username]`.
+  # An inverted reference is `Table`, standing in for the repository's table and any table related to it.
+  # A reference can carry filters from round brackets, `Posts(published: true)`, and be negated with `~Posts`.
+  class TableReference
+    FILTER_KEYWORD_MESSAGE = 'Filters go in round brackets, e.g. Posts(published: true)[:title], not %<keywords>s'
+
+    attr_reader :table, :filter, :negated
+
+    def initialize(table:, inverted: false, filter: DataExpression::Filter.new, negated: false)
+      @table = table
+      @inverted = inverted
+      @filter = filter
+      @negated = negated
+    end
+
+    # Each argument's type decides its meaning:
+    #   Posts[:title]                              # A symbol is a column. No columns selects every column.
+    #   Users[:username, Posts[:title]]            # A data expression is a nested table.
+    #   Reviews[:rating, author: Users[:username]] # A keyword with a data expression names a nested table.
+    def [](*arguments, **keywords)
+      raise ArgumentError, "~#{LowData.constant_name(table)} filters by related rows, so it can't select columns" if negated
+      return inverted(arguments:, keywords:) if @inverted
+
+      expressions, columns = arguments.partition { |argument| argument.is_a?(DataExpression) }
+      check_columns!(columns:)
+      nested = nested(pairs: expressions.map { |expression| [nil, expression] } + named(keywords:))
+
+      DataExpression.new(node: DataExpression::Source.new(table:, columns:, nested:, filter:))
+    end
+
+    # Each argument's type decides its meaning:
+    #   Posts(published: true)                     # A keyword with a value filters by that value.
+    #   Posts { views > 100 }                      # A block is a Sequel virtual row condition.
+    #   Users(Posts), Users(~Posts)                # A table keeps rows with, or without, related rows.
+    #   Users(Posts => 3..)                        # A table as a key keeps rows with that many related rows.
+    #   Reviews(author: Users(username: 'alice'))  # A keyword with a table names the foreign key to follow.
+    def where(*arguments, **keywords, &block)
+      related = arguments.map { |argument| related(key: nil, reference: argument) }
+      values = {}
+
+      keywords.each do |key, value|
+        if key.is_a?(TableReference) then related << related(key: nil, reference: key, count: value)
+        elsif value.is_a?(TableReference) then related << related(key:, reference: value)
+        else values[key] = value
+        end
+      end
+
+      conditions = block ? [Sequel.virtual_row(&block)] : []
+      with(filter: filter.merge(DataExpression::Filter.new(values:, conditions:, related:)))
+    end
+
+    # `~Posts` keeps rows without related posts.
+    def ~
+      with(negated: !negated)
+    end
+
+    def source
+      DataExpression::Source.new(table:, columns: [], filter:)
+    end
+
+    private
+
+    def with(filter: @filter, negated: @negated)
+      self.class.new(table:, inverted: @inverted, filter:, negated:)
+    end
+
+    def related(key:, reference:, count: nil)
+      unless reference.is_a?(TableReference)
+        raise ArgumentError, "Round brackets take filters and tables, e.g. Posts(published: true) or Users(Posts), not #{reference.inspect}"
+      end
+      raise ArgumentError, "Write ~#{LowData.constant_name(reference.table)} or a count, not both" if reference.negated && count
+
+      DataExpression::Related.new(key:, source: reference.source, count: count || (reference.negated ? 0 : 1..))
+    end
+
+    def named(keywords:)
+      named, filters = keywords.partition { |_key, value| value.is_a?(DataExpression) }
+      raise ArgumentError, format(FILTER_KEYWORD_MESSAGE, keywords: filters.to_h) if filters.any?
+
+      named
+    end
+
+    def nested(pairs:)
+      pairs.map { |key, expression| DataExpression::Nested.new(key:, node: expression.node) }
+    end
+
+    def inverted(arguments:, keywords:)
+      check_columns!(columns: arguments)
+      raise ArgumentError, 'Table[] takes column names only' if keywords.any?
+
+      DataExpression.new(node: DataExpression::Inversion.new(table:, columns: arguments, filter:))
+    end
+
+    def check_columns!(columns:)
+      invalid = columns.reject { |column| column.is_a?(Symbol) }
+      raise ArgumentError, "Columns are symbols, e.g. Posts[:title], not #{invalid.map(&:inspect).join(', ')}" if invalid.any?
+    end
+  end
+end

data/lib/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class LowData
+  VERSION = '0.1.0'
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: lowdata
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- maedi
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: expressions
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: lowtype
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: The repository pattern with data expressions + the usual query building
+email:
+- maediprichard@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/col.rb
+- lib/data_expression.rb
+- lib/errors.rb
+- lib/filter_compiler.rb
+- lib/inversion.rb
+- lib/lowdata.rb
+- lib/nesting.rb
+- lib/nesting_tree.rb
+- lib/order_compiler.rb
+- lib/query_compiler.rb
+- lib/rows.rb
+- lib/schema.rb
+- lib/table_reference.rb
+- lib/version.rb
+homepage: https://github.com/low-rb/lowdata
+licenses: []
+metadata:
+  homepage_uri: https://github.com/low-rb/lowdata
+  source_code_uri: https://github.com/low-rb/lowdata/src/branch/main
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.6
+specification_version: 4
+summary: The repository pattern with a twist; Data Expressions
+test_files: []