sequel-notion 0.2.0

data/lib/sequel/notion/dataset.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "sequel/notion/dataset_aggregates"
+require "sequel/notion/dataset_compounds"
+require "sequel/notion/dataset_computed"
+require "sequel/notion/dataset_grouping"
+require "sequel/notion/dataset_joins"
+require "sequel/notion/dataset_pages"
+require "sequel/notion/dataset_selection"
+require "sequel/notion/dataset_truncation"
+require "sequel/notion/type_map"
+
+module Sequel
+    module Notion
+        class Dataset < Sequel::Dataset
+            include DatasetAggregates
+            include DatasetCompounds
+            include DatasetComputed
+            include DatasetGrouping
+            include DatasetJoins
+            include DatasetPages
+            include DatasetSelection
+            include DatasetTruncation
+
+            def columns
+                computed_columns || selection&.map(&:last) ||
+                    db.schema(source_table).map(&:first)
+            end
+
+            def columns! = columns
+
+            # Sequel renders SQL before fetching; there is none to render
+            def select_sql = "NOTION #{compound? ? "compound" : source_table}"
+
+            # Computed in Ruby, the first row of each key
+            def supports_distinct_on? = true
+
+            # Sequel's cached loaders swap WHERE for SQL placeholders,
+            # which only a SQL database can fill in
+            def supports_placeholder_literalizer? = false
+
+            # Rows, auto-paginated, honouring LIMIT, OFFSET and SELECT
+            def fetch_rows(sql)
+                if @opts[:sql] || sql != select_sql
+                    raise Error, "Notion datasets take no SQL"
+                end
+
+                budgeted { each_row { yield it } }
+            end
+
+            # Streams through Notion's cursor, as cursor adapters do: no
+            # order needed, one request per page of rows_per_fetch (at
+            # most 100). Sequel's :strategy, which pages by OFFSET or by
+            # filtering on the order columns, does not apply.
+            def paged_each(opts = OPTS, &)
+                return enum_for(:paged_each, opts) unless block_given?
+
+                size = opts[:rows_per_fetch]
+                (size ? clone(notion_page_size: size) : self).each(&)
+                self
+            end
+
+            def empty? = limit(1).first.nil?
+
+            # ----------------------------------------------------------
+            # Insert → create page
+            # ----------------------------------------------------------
+
+            def insert(*values)
+                row = insert_hash(values).transform_keys(&:to_sym)
+                row.delete(:id)
+                if row.delete(:in_trash)
+                    raise Error, "a page cannot be created in the trash"
+                end
+
+                db.notion_create_page(data_source_id, properties_for(row))["id"]
+            end
+
+            def import(columns, values, _opts = OPTS)
+                values.map { insert(columns.zip(it).to_h) }
+            end
+
+            def multi_insert(hashes, _opts = OPTS) = hashes.map { insert(it) }
+
+            # ----------------------------------------------------------
+            # Update / delete — ids are collected first, so that pages
+            # leaving the filter while being patched are not skipped
+            # ----------------------------------------------------------
+
+            def update(values = OPTS)
+                raise Error, "update takes a Hash" unless values.is_a?(Hash)
+
+                values = values.transform_keys(&:to_sym)
+                in_trash = values.delete(:in_trash)
+                values.delete(:id)
+                props = properties_for(values)
+
+                page_ids.each do |id|
+                    db.notion_update_page(id, props, in_trash:)
+                end.size
+            end
+
+            def delete
+                page_ids.each { db.notion_trash_page(it) }.size
+            end
+
+            private
+
+            def page_rows
+                sel = selection
+                each_notion_page do |page|
+                    yield project(TypeMap.page_to_row(complete_page(page, sel)),
+                                  sel)
+                end
+            end
+
+            # One query's requests, under client_side's max_requests
+            def budgeted(&) = db.with_request_budget(@opts[:max_requests], &)
+
+            def source_table
+                from = @opts[:from]
+                unless from&.size == 1 && from.first.is_a?(Symbol)
+                    raise Error, "Notion datasets need a single table name"
+                end
+
+                from.first
+            end
+
+            def data_source_id
+                db.data_source_id_for(source_table) or
+                    raise Error, "Unknown data source: #{source_table}"
+            end
+
+            def properties_for(values)
+                types = db.property_type_map(data_source_id)
+                TypeMap.row_to_properties(values, types)
+            end
+
+            # Columns a positional insert fills, in schema order
+            def writable
+                db.schema(source_table)
+                  .reject { |_, info| info[:generated] }
+                  .map(&:first) - Schema::PAGE_COLUMNS.map(&:first)
+            end
+
+            def positional(vals)
+                cols = writable
+                if vals.size > cols.size
+                    raise Error, "#{vals.size} values for #{cols.size} " \
+                                 "writable columns"
+                end
+
+                cols.take(vals.size).zip(vals).to_h
+            end
+
+            def insert_hash(values)
+                case values
+                in [] then {}
+                in [Hash => hash] then hash
+                in [Array => cols, Array => vals] then cols.zip(vals).to_h
+                in [Array => vals] then positional(vals)
+                else raise Error, "Unsupported insert arguments"
+                end
+            end
+        end
+    end
+end

data/lib/sequel/notion/dataset_aggregates.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "sequel/notion/filter_compiler"
+
+module Sequel
+    module Notion
+        # Aggregates and DISTINCT, which Notion's API does not compute:
+        # worked out in Ruby over the rows the query returns, so their
+        # cost is that of reading every one. As in SQL, an aggregate skips
+        # NULLs and is NULL over no value, and DISTINCT comes before
+        # OFFSET and LIMIT.
+        module DatasetAggregates
+            AGGREGATES = {
+                sum: ->(v) { v.sum unless v.empty? },
+                avg: ->(v) { v.sum.fdiv(v.size) unless v.empty? },
+                min: :min.to_proc,
+                max: :max.to_proc
+            }.freeze
+
+            # Rows, or the non-NULL values of one column
+            def count(*args, &block)
+                if block || args.size > 1
+                    raise Error, "Notion datasets count rows or one column"
+                end
+
+                return budgeted { count_rows } if args.empty?
+
+                client_side!("count(#{args.first.inspect})")
+                budgeted { column_values(args.first).size }
+            end
+
+            private
+
+            def count_rows
+                return to_enum(:fetch_rows, select_sql).count if computed?
+
+                n = 0
+                each_notion_page { n += 1 }
+                n
+            end
+
+            # Sequel's sum, avg, min and max all land here
+            def _aggregate(function, arg)
+                client_side!(function.to_s)
+                values = budgeted { column_values(arg) }
+                AGGREGATES.fetch(function).call(values)
+            rescue TypeError, ArgumentError
+                raise Error, "cannot compute #{function} of #{arg.inspect}"
+            end
+
+            def distinct_on? = !@opts[:distinct].empty?
+
+            def distinct_source(select = @opts[:select])
+                clone(distinct: nil, limit: nil, offset: nil, select:)
+                    .naked.all
+            end
+
+            def first_of_each_key
+                keys = @opts[:distinct].map { group_column(it) }
+                sel  = selection
+                distinct_source(nil).uniq { |row| keys.map { row[it] } }
+                                    .map { project(it, sel) }
+            end
+
+            # A column's values, read under a hidden name so a qualified
+            # column keeps its table
+            def column_values(arg)
+                FilterCompiler.property_name(arg)
+                naked.select(Sequel.as(arg, :__value)).map(:__value).compact
+            end
+
+            # Every row first, then OFFSET and LIMIT over the distinct
+            # ones; DISTINCT ON keeps the first row of each key
+            def distinct_rows(&)
+                rows = distinct_on? ? first_of_each_key : distinct_source.uniq
+                rows = rows.drop(@opts[:offset] || 0)
+                rows = rows.take(@opts[:limit]) if @opts[:limit]
+                rows.each(&)
+            end
+        end
+    end
+end

data/lib/sequel/notion/dataset_compounds.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # UNION, INTERSECT and EXCEPT, which Notion's API does not
+        # compute: each query's rows are read, then combined in Ruby as
+        # SQL does (UNION ALL keeps repeats, the others drop them).
+        # Sequel wraps the left query in a subquery; ORDER, OFFSET,
+        # LIMIT and a SELECT of columns on the result apply to the
+        # combined rows.
+        module DatasetCompounds
+            # Clauses the wrapping query cannot add to a combination
+            OUTER_UNSUPPORTED = %i[where group having distinct join].freeze
+
+            private
+
+            # A subquery that combines nothing (from_self) is no compound,
+            # and gets source_table's refusal
+            def compound?
+                from = @opts[:from]
+                from&.size == 1 && from.first.is_a?(SQL::AliasedExpression) &&
+                    from.first.expression.is_a?(Notion::Dataset) &&
+                    !from.first.expression.opts[:compounds].nil?
+            end
+
+            def compound_inner = @opts[:from].first.expression
+
+            def compound_rows
+                bad = OUTER_UNSUPPORTED.select { @opts[it] }
+                unless bad.empty?
+                    raise Error, "a combined query takes no #{bad.join(", ")}"
+                end
+
+                sel  = compound_selection
+                rows = sort_rows(combined_rows).drop(@opts[:offset] || 0)
+                rows = rows.take(@opts[:limit]) if @opts[:limit]
+                rows.each { yield project(it, sel) }
+            end
+
+            def compound_columns
+                compound_selection&.map(&:last) || compound_inner.columns
+            end
+
+            # The outer SELECT, of columns the combined rows have
+            def compound_selection
+                sel = @opts[:select]
+                return if sel.nil? || sel.empty?
+                return if sel.any? { it == :* || it.is_a?(SQL::ColumnAll) }
+
+                pairs   = sel.map { selected(it) }
+                unknown = pairs.map(&:first) - compound_inner.columns
+                return pairs if unknown.empty?
+
+                raise Error, "Unknown columns in select: #{unknown.join(", ")}"
+            end
+
+            def combined_rows
+                first = compound_inner.clone(compounds: nil)
+                names = first.columns
+                rows  = first.naked.all
+                compound_inner.opts[:compounds].reduce(rows) do |left, step|
+                    op, other, all = step
+                    combined(op, left, renamed(other, names), all)
+                end
+            end
+
+            # Another query's rows under the first query's column names,
+            # matched by position, as SQL names a combination's columns
+            def renamed(other, names)
+                columns = other.columns
+                unless columns.size == names.size
+                    raise Error, "combined queries select #{names.size} " \
+                                 "and #{columns.size} columns"
+                end
+
+                other.naked.all.map do |row|
+                    names.zip(columns).to_h { |name, col| [name, row[col]] }
+                end
+            end
+
+            def combined(op, left, right, all)
+                return all ? left + right : (left + right).uniq if op == :union
+                raise Error, "#{op.upcase} ALL is not supported" if all
+
+                op == :intersect ? left.uniq & right : left.uniq - right
+            end
+        end
+    end
+end

data/lib/sequel/notion/dataset_computed.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # Rows Notion's API cannot compute (combined queries, groups,
+        # DISTINCT, joins), worked out in Ruby by DatasetCompounds,
+        # DatasetGrouping, DatasetAggregates and DatasetJoins; ORDER over
+        # such rows is sorted here, empty values last, as Notion sorts
+        # them. Such a query reads every row it matches, so it runs only
+        # on a dataset that opts in with client_side; otherwise it raises
+        # before the first request.
+        module DatasetComputed
+            # Allow what Notion cannot compute to be computed in Ruby;
+            # max_requests caps the Notion requests each query may send
+            def client_side(max_requests: nil)
+                clone(client_side: true, max_requests:)
+            end
+
+            private
+
+            # Sequel wraps the left query of a UNION, which keeps the opt-in
+            def client_side?
+                @opts[:client_side] ||
+                    (compound? && compound_inner.opts[:client_side])
+            end
+
+            def client_side!(what)
+                return if client_side?
+
+                raise Error, "#{what} is computed in Ruby over every row " \
+                             "the query returns; call client_side on the " \
+                             "dataset to allow it"
+            end
+
+            def computed_operation
+                compounds = compound_inner.opts[:compounds] if compound?
+                return compounds.first.first.upcase.to_s if compounds
+                return "GROUP BY" if @opts[:group]
+                return "DISTINCT" if @opts[:distinct]
+
+                "JOIN"
+            end
+
+            # Rows Notion cannot compute, worked out in Ruby
+            def computed?
+                compound? || @opts[:group] || @opts[:distinct] || joined?
+            end
+
+            # Groups and DISTINCT over joined rows: the join comes last
+            def computed_rows(&)
+                raise Error, "Notion datasets do not support: lock" if
+                    @opts[:lock]
+
+                client_side!(computed_operation)
+                return compound_rows(&) if compound?
+                return grouped_rows(&) if @opts[:group]
+                return distinct_rows(&) if @opts[:distinct]
+
+                join_rows(&)
+            end
+
+            def computed_columns
+                return compound_columns if compound?
+                return grouped_outputs.map(&:first) if @opts[:group]
+
+                join_columns_out(join_sources) if joined?
+            end
+
+            # The query's rows, computed in Ruby or read from Notion, each
+            # given to the caller outside the query's request budget
+            def each_row
+                give = ->(row) { db.outside_request_budget { yield row } }
+                computed? ? computed_rows(&give) : page_rows(&give)
+            end
+
+            # Pages fetched one by one come in the list's order: ORDER
+            # sorts them here, as Notion sorts a query's, after the sort
+            # compiler has refused what Notion would
+            def ordered_pages(pages)
+                SortCompiler.compile(@opts[:order])
+                rows = pages.map { TypeMap.page_to_row(it).merge(__page: it) }
+                sort_rows(rows).map { it[:__page] }
+            end
+
+            # ORDER over rows already computed (groups, combined rows)
+            def sort_rows(rows)
+                order = Array(@opts[:order]).map { grouped_order(it) }
+                return rows if order.empty?
+
+                rows.sort do |a, b|
+                    order.lazy.map { |col, desc| compare(a[col], b[col], desc) }
+                         .find(&:nonzero?) || 0
+                end
+            end
+
+            def grouped_order(clause)
+                if clause.is_a?(SQL::OrderedExpression)
+                    return [group_column(clause.expression), clause.descending]
+                end
+
+                [group_column(clause), false]
+            end
+
+            def compare(left, right, desc)
+                return (left.nil? ? 0 : -1) if right.nil?
+                return 1 if left.nil?
+
+                order = sortable(left) <=> sortable(right) or
+                    raise Error, "cannot order #{left.inspect} and " \
+                                 "#{right.inspect}"
+                desc ? -order : order
+            end
+
+            # A checkbox sorts false first, as Notion sorts it
+            def sortable(value)
+                case value
+                when false then 0
+                when true then 1
+                else value
+                end
+            end
+        end
+    end
+end

data/lib/sequel/notion/dataset_grouping.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require "sequel/notion/dataset_having"
+require "sequel/notion/filter_compiler"
+require "sequel/notion/group_accumulator"
+
+module Sequel
+    module Notion
+        # GROUP BY, which Notion's API does not compute: the rows the
+        # query returns are read once, each group keeping one running
+        # value per aggregate; ORDER, OFFSET and LIMIT then apply to the
+        # groups. Empty values sort last, as in Notion.
+        module DatasetGrouping
+            include DatasetHaving
+
+            private
+
+            def grouped_rows(&)
+                rows = sort_rows(groups).drop(@opts[:offset] || 0)
+                rows = rows.take(@opts[:limit]) if @opts[:limit]
+                rows.each(&)
+            end
+
+            # One row per group, HAVING applied, hidden outputs dropped
+            def groups
+                outs  = grouped_outputs
+                funcs = having_functions
+                rows  = group_rows(outs + having_outputs(funcs))
+                rows  = rows.select { having?(it, funcs) } if @opts[:having]
+                rows.map { it.slice(*outs.map(&:first)) }
+            end
+
+            def group_rows(outs)
+                slots = group_slots(outs)
+                accumulate(outs, slots).map do |key, accs|
+                    grouped_row(outs, key, accs.each)
+                end
+            end
+
+            def having_outputs(funcs)
+                funcs.each_with_index.map do |func, i|
+                    aggregate_output(func, having_name(i))
+                end
+            end
+
+            def group_column(expr) = FilterCompiler.property_name(expr).to_sym
+
+            # A column's identity: its table when qualified, and its name
+            def column_key(expr)
+                return [expr.table.to_s, expr.column.to_sym] if
+                    expr.is_a?(SQL::QualifiedIdentifier)
+
+                [nil, group_column(expr)]
+            end
+
+            def group_keys = @opts[:group].map { column_key(it) }
+
+            # [name, :key, column] or [name, function, column (nil for *)]
+            def grouped_outputs
+                (@opts[:select] || @opts[:group]).map { group_output(it) }
+            end
+
+            def group_output(expr)
+                name = expr.alias.to_sym if expr.is_a?(SQL::AliasedExpression)
+                expr = expr.expression if name
+                return aggregate_output(expr, name) if expr.is_a?(SQL::Function)
+
+                key = column_key(expr)
+                unless group_keys.include?(key)
+                    raise Error, "#{key.last} is neither grouped nor aggregated"
+                end
+
+                [name || key.last, :key, expr]
+            end
+
+            def aggregate_output(func, name)
+                function = func.name
+                function = function.value if function.is_a?(SQL::Identifier)
+                function = function.to_s.downcase.to_sym
+                [name || function, function, aggregate_column(func, function)]
+            end
+
+            # The column aggregated, or nil for count(*)
+            def aggregate_column(func, function)
+                arg = func.args.first
+                if plain_aggregate?(func, function)
+                    return arg.tap { column_key(it) } unless star?(func)
+                    return if function == :count
+                end
+
+                raise Error, "Unsupported aggregate: #{func.inspect}"
+            end
+
+            def star?(func)
+                func.opts[:*] || func.args.empty? || func.args.first == "*"
+            end
+
+            def plain_aggregate?(func, function)
+                GroupAccumulator::FUNCTIONS.include?(function) &&
+                    func.args.size <= 1 && (func.opts.keys - [:*]).empty?
+            end
+
+            # column key => [column, hidden name it is read under]
+            def group_slots(outs)
+                columns = @opts[:group] + outs.filter_map(&:last)
+                columns.uniq { column_key(it) }.each_with_index
+                       .to_h { |col, i| [column_key(col), [col, :"__c#{i}"]] }
+            end
+
+            def accumulate(outs, slots)
+                aggs  = outs.reject { it[1] == :key }
+                names = group_keys.map { slots[it].last }
+                group_source(slots).each_with_object({}) do |row, groups|
+                    accs = groups[names.map { row[it] }] ||= accumulators(aggs)
+                    add_row(row, aggs.zip(accs), slots)
+                end
+            end
+
+            def accumulators(aggs) = aggs.map { GroupAccumulator.new(it[1]) }
+
+            def add_row(row, pairs, slots)
+                pairs.each do |(_, _, col), acc|
+                    acc.add(col ? row[slots[column_key(col)].last] : 1)
+                end
+            end
+
+            # The rows to group: the WHERE kept, Notion asked for nothing
+            # else, and only the columns the groups need
+            def group_source(slots)
+                clone(group: nil, select: nil, order: nil, limit: nil,
+                      offset: nil, distinct: nil, having: nil)
+                    .naked.select(*slots.values.map { Sequel.as(*it) })
+            end
+
+            def grouped_row(outs, key, accs)
+                keys = group_keys
+                outs.to_h do |name, kind, col|
+                    next [name, accs.next.value] unless kind == :key
+
+                    [name, key[keys.index(column_key(col))]]
+                end
+            end
+        end
+    end
+end