sequel-notion 0.2.0

data/lib/sequel/notion/filter_nulls.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # SQL's rule for empty values (rule E), mixed into FilterCompiler
+        # and applied once, to the finished filter. Notion's
+        # does_not_equal and does_not_contain match an empty property;
+        # SQL's != and NOT LIKE never match NULL. Each such leaf gets an
+        # is_not_empty beside it: spliced into an enclosing "and", so
+        # nesting does not grow, else wrapped in one. It runs after
+        # negation, which would otherwise turn the guard into is_empty.
+        module FilterNulls
+            NEGATIVE_OPS = %w[does_not_equal does_not_contain].freeze
+
+            # Never empty (checkbox, unique_id), or already excluding
+            # empty results (formula, checked live). A rollup is not: its
+            # does_not_equal matches an empty one (checked live), so it is
+            # guarded under its kind.
+            UNGUARDED = %w[checkbox unique_id formula].freeze
+
+            private
+
+            def sql_nulls(filter) = conjunction(guarded(filter))
+
+            # The conditions an "and" holding +filter+ needs
+            def guarded(filter)
+                if filter.key?("and")
+                    [{ "and" => filter["and"].flat_map { guarded(it) }.uniq }]
+                elsif filter.key?("or")
+                    [{ "or" => filter["or"].map { conjunction(guarded(it)) } }]
+                else
+                    guarded_leaf(filter)
+                end
+            end
+
+            def guarded_leaf(leaf)
+                key  = (leaf.keys - ["property"]).first
+                kind = leaf[key].keys.first if key == "rollup"
+                cond = kind ? leaf[key][kind] : leaf[key]
+                return [leaf] if UNGUARDED.include?(key) ||
+                                 !NEGATIVE_OPS.include?(cond.keys.first)
+
+                guard = { "is_not_empty" => true }
+                [leaf, { "property" => leaf["property"],
+                         key => kind ? { kind => guard } : guard }]
+            end
+
+            def conjunction(conds)
+                conds.one? ? conds.first : { "and" => conds }
+            end
+        end
+    end
+end

data/lib/sequel/notion/filter_predicates.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require "sequel/notion/filter_nested"
+require "sequel/notion/filter_tables"
+
+module Sequel
+    module Notion
+        # Predicate compilers mixed into FilterCompiler: emptiness checks,
+        # IS / IS NOT, bare checkbox columns, and IN / NOT IN. All private.
+        module FilterPredicates
+            # Keys whose IN goes value by value: nested, or a date, which
+            # has no does_not_equal
+            IN_BY_VALUE = (FilterNested::NESTED_KEYS + ["date"]).freeze
+
+            private
+
+            # ----------------------------------------------------------
+            # Null / empty check  ->  is_empty / is_not_empty
+            # ----------------------------------------------------------
+            #
+            # Single method (dedupe rule) used both for `= nil` / `!= nil`
+            # and for `IS nil` / `IS NOT nil`.
+            def null_check_filter(name, empty:)
+                type = lookup_type!(name)
+                key = filter_key_for!(name, type)
+
+                op = empty ? "is_empty" : "is_not_empty"
+                return nested_null_filter(name, key, op) if
+                    FilterNested::NESTED_KEYS.include?(key)
+
+                ensure_supported!(name, key, op)
+
+                { "property" => name, key => { op => true } }
+            end
+
+            # ----------------------------------------------------------
+            # IS / IS NOT (rule I)
+            # ----------------------------------------------------------
+
+            def compile_is(expr)
+                left, value = expr.args
+                name    = FilterCompiler.property_name(left)
+                negated = expr.op == :"IS NOT"
+
+                case value
+                when nil then null_check_filter(name, empty: !negated)
+                when true, false then compile_is_boolean(name, negated, value)
+                else
+                    raise unsupported_is_value_error(name, value)
+                end
+            end
+
+            def unsupported_is_value_error(name, value)
+                Sequel::Error.new(
+                    "Unsupported IS value for property '#{name}': " \
+                    "#{value.inspect}"
+                )
+            end
+
+            # A checkbox, or a formula through its nested checkbox key, as
+            # for a bare column
+            def compile_is_boolean(name, negated, value)
+                type = lookup_type!(name)
+                case filter_key_for!(name, type)
+                when "checkbox"
+                    op = negated ? "does_not_equal" : "equals"
+                    { "property" => name, "checkbox" => { op => value } }
+                when "formula"
+                    op = negated ? :"!=" : :"="
+                    compile_formula_comparison(name, op, value)
+                else raise boolean_type_error(name, type, value)
+                end
+            end
+
+            def boolean_type_error(name, type, value)
+                Sequel::Error.new("'#{name}' IS #{value}: a checkbox or " \
+                                  "formula property only, got type '#{type}'")
+            end
+
+            def contains_type?(key)
+                FilterTables::CONTAINS_TYPES.include?(key)
+            end
+
+            # ----------------------------------------------------------
+            # Bare column (rule B)
+            # ----------------------------------------------------------
+
+            # A checkbox, or a formula through its checkbox key
+            def compile_bare_column(expr, value: true)
+                name = FilterCompiler.property_name(expr)
+                type = lookup_type!(name)
+                case filter_key_for!(name, type)
+                when "checkbox"
+                    { "property" => name, "checkbox" => { "equals" => value } }
+                when "formula"
+                    compile_formula_comparison(name, :"=", value)
+                else raise bare_column_error(name, type)
+                end
+            end
+
+            def bare_column_error(name, type)
+                Sequel::Error.new("Bare column '#{name}' must be a checkbox " \
+                                  "or formula property, got type '#{type}'")
+            end
+
+            # ----------------------------------------------------------
+            # IN / NOT IN (rule IN)
+            # ----------------------------------------------------------
+
+            def compile_in(expr)
+                left, value = expr.args
+                name = FilterCompiler.property_name(left)
+                validate_in_array!(value, name)
+
+                key = filter_key_for!(name, lookup_type!(name))
+                return empty_in(expr.op) if value.empty?
+                return in_by_value(expr.op, name, value, key) if
+                    IN_BY_VALUE.include?(key)
+
+                base_op = in_base_operator(expr.op, key)
+                ensure_supported!(name, key, base_op)
+
+                in_filter(expr.op, name, key, base_op, value)
+            end
+
+            def validate_in_array!(value, name)
+                return if value.is_a?(Array)
+
+                raise Sequel::Error,
+                      "IN/NOT IN requires an Array value for " \
+                      "property '#{name}', got #{value.class}"
+            end
+
+            # IN () matches no page, NOT IN () every page
+            def empty_in(sequel_op)
+                return FilterConstants::NOTHING if sequel_op == :IN
+
+                FilterConstants::EVERYTHING
+            end
+
+            def in_base_operator(sequel_op, key)
+                base_op = sequel_op == :IN ? "equals" : "does_not_equal"
+                contains_type?(key) ? equal_to_contains(base_op) : base_op
+            end
+
+            # A nil in the list means empty, as `where(P: nil)` does
+            def in_filter(sequel_op, name, key, base_op, value)
+                leaves = value.map do |v|
+                    next null_check_filter(name, empty: sequel_op == :IN) if
+                        v.nil?
+
+                    { "property" => name,
+                      key => { base_op => coerce_value(v, key) } }
+                end
+
+                { (sequel_op == :IN ? "or" : "and") => leaves }
+            end
+        end
+    end
+end

data/lib/sequel/notion/filter_shape.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # The finished filter, fitted to Notion's shape (rule K), mixed
+        # into FilterCompiler. Notion nests and/or two levels deep at
+        # most. An "and" directly inside an "and" (or an "or" inside an
+        # "or") is spliced into its parent; a filter still too deep has
+        # its inner level distributed, (a & b) | c becoming
+        # (a | c) & (b | c), up to MAX_CLAUSES clauses; what remains too
+        # deep raises rather than being sent to fail with a 400.
+        module FilterShape
+            MAX_DEPTH   = 2
+            MAX_CLAUSES = 32
+
+            private
+
+            def notion_shape(filter)
+                flat = flattened(filter)
+                flat = distributed(flat) if compound_depth(flat) > MAX_DEPTH
+                return flat if compound_depth(flat) <= MAX_DEPTH
+
+                raise too_deep(flat)
+            end
+
+            def flattened(filter)
+                op = compound_op(filter)
+                return filter unless op
+
+                { op => filter[op].flat_map do |kid|
+                    kid = flattened(kid)
+                    compound_op(kid) == op ? kid[op] : [kid]
+                end }
+            end
+
+            # Each kid two levels deep, whose op is the other one,
+            # rewritten in the root's op and spliced into the root
+            def distributed(filter)
+                op = compound_op(filter)
+                { op => filter[op].flat_map do |kid|
+                    compound_depth(kid) == MAX_DEPTH ? swapped(kid, op) : [kid]
+                end }
+            end
+
+            # The clauses of kid, an "or" of "and"s (or the reverse),
+            # rewritten as op's children
+            def swapped(kid, op)
+                kop     = compound_op(kid)
+                choices = kid[kop].map { compound_op(it) == op ? it[op] : [it] }
+                clauses = choices.first.product(*choices.drop(1))
+                raise too_deep(kid) if clauses.size > MAX_CLAUSES
+
+                clauses.map { conjoined(kop, it.uniq) }.uniq
+            end
+
+            def conjoined(op, conds)
+                conds.one? ? conds.first : { op => conds }
+            end
+
+            def too_deep(filter)
+                Sequel::Error.new("Notion nests and/or at most #{MAX_DEPTH} " \
+                                  "levels deep: #{filter.inspect}")
+            end
+
+            def compound_depth(filter)
+                op = compound_op(filter)
+                op ? 1 + filter[op].map { compound_depth(it) }.max : 0
+            end
+
+            def compound_op(filter)
+                %w[and or].find { filter.key?(it) }
+            end
+        end
+    end
+end

data/lib/sequel/notion/filter_tables.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # Static lookup tables shared by the filter-compiling mixins
+        # (FilterHelpers, FilterPredicates, FilterLike, FilterComparison,
+        # FilterNegation). Pure data, no behaviour.
+        module FilterTables
+            OPERATOR_MAP = {
+                :"=" => "equals",
+                :"!=" => "does_not_equal",
+                :> => "greater_than",
+                :< => "less_than",
+                :>= => "greater_than_or_equal_to",
+                :<= => "less_than_or_equal_to"
+            }.freeze
+
+            # Mirrors an operator when the comparison's operands are swapped
+            # (rule S): a < b  ==  b > a
+            MIRROR_OPERATOR = {
+                :< => :>,
+                :> => :<,
+                :<= => :>=,
+                :>= => :<=,
+                :"=" => :"=",
+                :"!=" => :"!="
+            }.freeze
+
+            FILTER_TYPE_KEY = {
+                "title" => "rich_text",
+                "rich_text" => "rich_text",
+                "url" => "rich_text",
+                "email" => "rich_text",
+                "phone_number" => "phone_number",
+                "created_time" => "date",
+                "last_edited_time" => "date",
+                "date" => "date",
+                "number" => "number",
+                "select" => "select",
+                "multi_select" => "multi_select",
+                "status" => "status",
+                "checkbox" => "checkbox",
+                "people" => "people",
+                "relation" => "relation",
+                "files" => "files",
+                "formula" => "formula",
+                "unique_id" => "unique_id",
+                "rollup" => "rollup"
+            }.freeze
+
+            RICH_TEXT_OPS = %w[equals does_not_equal contains does_not_contain
+                               starts_with ends_with is_empty
+                               is_not_empty].freeze
+
+            SUPPORTED_OPS = {
+                "rich_text" => RICH_TEXT_OPS,
+                "string" => RICH_TEXT_OPS, # formula inner key for String values
+                "number" => %w[equals does_not_equal greater_than less_than
+                               greater_than_or_equal_to
+                               less_than_or_equal_to is_empty is_not_empty],
+                "select" => %w[equals does_not_equal is_empty is_not_empty],
+                "multi_select" => %w[contains does_not_contain is_empty
+                                     is_not_empty],
+                "status" => %w[equals does_not_equal is_empty is_not_empty],
+                "date" => %w[equals before after on_or_before on_or_after
+                             is_empty is_not_empty],
+                "checkbox" => %w[equals does_not_equal],
+                "phone_number" => %w[equals does_not_equal contains
+                                     does_not_contain starts_with ends_with
+                                     is_empty is_not_empty],
+                "people" => %w[contains does_not_contain is_empty
+                               is_not_empty],
+                "relation" => %w[contains does_not_contain is_empty
+                                 is_not_empty],
+                "files" => %w[is_empty is_not_empty],
+                "unique_id" => %w[equals does_not_equal greater_than less_than
+                                  greater_than_or_equal_to
+                                  less_than_or_equal_to]
+            }.freeze
+
+            DATE_OPERATOR_MAP = {
+                "greater_than" => "after",
+                "less_than" => "before",
+                "greater_than_or_equal_to" => "on_or_after",
+                "less_than_or_equal_to" => "on_or_before"
+            }.freeze
+
+            # Explicit inverse table for negation (rule G).
+            NEGATE_MAP = {
+                "equals" => "does_not_equal",
+                "does_not_equal" => "equals",
+                "contains" => "does_not_contain",
+                "does_not_contain" => "contains",
+                "is_empty" => "is_not_empty",
+                "is_not_empty" => "is_empty",
+                "greater_than" => "less_than_or_equal_to",
+                "less_than_or_equal_to" => "greater_than",
+                "less_than" => "greater_than_or_equal_to",
+                "greater_than_or_equal_to" => "less_than",
+                "before" => "on_or_after",
+                "on_or_after" => "before",
+                "after" => "on_or_before",
+                "on_or_before" => "after"
+            }.freeze
+
+            # Property types whose (not)equal comparisons are expressed
+            # through contains/does_not_contain instead (rule IN).
+            CONTAINS_TYPES = %w[multi_select people relation].freeze
+        end
+    end
+end

data/lib/sequel/notion/group_accumulator.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # One aggregate over one group, kept as a running value rather
+        # than the group's rows: count, sum, avg, min or max of the
+        # non-NULL values, as SQL defines them. count(*) is a count of a
+        # value given once per row.
+        class GroupAccumulator
+            FUNCTIONS = %i[count sum avg min max].freeze
+
+            def initialize(function)
+                @function = function
+                @count    = 0
+                @value    = nil
+            end
+
+            def add(value)
+                return if value.nil?
+
+                numeric!(value)
+                @count += 1
+                @value = @value.nil? ? value : combined(value)
+            rescue ArgumentError
+                raise Error, "cannot compute #{@function} of #{value.inspect}"
+            end
+
+            def value
+                case @function
+                when :count then @count
+                when :avg then @value&.fdiv(@count)
+                else @value
+                end
+            end
+
+            private
+
+            def numeric!(value)
+                return if value.is_a?(Numeric)
+                return unless %i[sum avg].include?(@function)
+
+                raise Error, "cannot compute #{@function} of #{value.inspect}"
+            end
+
+            def combined(value)
+                case @function
+                when :sum, :avg then @value + value
+                when :min then [@value, value].min
+                when :max then [@value, value].max
+                else @value
+                end
+            end
+        end
+    end
+end

data/lib/sequel/notion/join_output.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "sequel/notion/filter_compiler"
+
+module Sequel
+    module Notion
+        # A joined row ({ source name => row }) as the row a query
+        # returns, mixed into DatasetJoins: the selected columns, or every
+        # column with a later table winning a shared name.
+        module JoinOutput
+            private
+
+            def join_output(row, sources)
+                sel = @opts[:select]
+                return merged_row(row, sources) if sel.nil? || sel.empty?
+
+                sel.to_h { join_selected(it, row, sources) }
+            end
+
+            def merged_row(row, sources)
+                sources.each_with_object({}) do |(name, table), out|
+                    out.merge!(row[name] || null_row(table))
+                end
+            end
+
+            def null_row(table) = db.schema(table).to_h { [it.first, nil] }
+
+            def join_selected(expr, row, sources)
+                name = expr.alias.to_sym if expr.is_a?(SQL::AliasedExpression)
+                expr = expr.expression if name
+                source, column = join_column(expr, sources)
+                [name || column, row[source]&.[](column)]
+            end
+
+            def join_column(expr, sources)
+                case expr
+                when SQL::QualifiedIdentifier
+                    [qualified(expr, sources), expr.column.to_sym]
+                when Symbol, SQL::Identifier
+                    [owner(expr, sources),
+                     FilterCompiler.property_name(expr).to_sym]
+                else
+                    raise Error, "only columns can be selected from a " \
+                                 "join: #{expr.inspect}"
+                end
+            end
+
+            def join_columns_out(sources)
+                sel = @opts[:select]
+                return sel.map { join_selected(it, {}, sources).first } if
+                    sel && !sel.empty?
+
+                sources.flat_map { |_, table| db.schema(table).map(&:first) }
+                       .uniq
+            end
+        end
+    end
+end

data/lib/sequel/notion/join_where.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "sequel/notion/filter_compiler"
+
+module Sequel
+    module Notion
+        # Which side of a join each column and each WHERE condition
+        # belongs to, mixed into DatasetJoins. A source is
+        # [name, table]: the alias, or the table's own name. An
+        # unqualified column belongs to the one source whose schema has
+        # it. A condition testing one source runs in Notion, on it; one
+        # testing two has no Notion filter and raises.
+        #
+        # WHERE runs after the join. On an inner join, filtering a side
+        # first gives the same rows; on a left join it does not, since a
+        # row whose partner fails the condition would be kept with an
+        # empty side. So a left join's own conditions filter its partners
+        # after matching on the whole table, and a row with no partner is
+        # kept only if an empty page passes them, as NULLs do in SQL.
+        module JoinWhere
+            private
+
+            def left_joined(rows, left, right, source, wheres)
+                name, = source
+                index = join_index(source_rows(source, {}), right)
+                kept  = source_rows(source, wheres).to_set { it[:id] }
+                rows.flat_map do |row|
+                    found = partners(row, *left, index)
+                    next empty_side(row, source, wheres) if found.empty?
+
+                    found.select { kept.include?(it[:id]) }
+                         .map { row.merge(name => it) }
+                end
+            end
+
+            def empty_side(row, (name, table), wheres)
+                cond   = wheres[name].reduce(db[table]) { |ds, c| ds.where(c) }
+                filter = FilterCompiler.for(db, db.data_source_id_for(table))
+                                       .compile(cond.opts[:where])
+                empty_match?(filter) ? [row.merge(name => nil)] : []
+            end
+
+            # Whether an empty page passes a compiled filter: only an
+            # is_empty test does (negations carry is_not_empty)
+            def empty_match?(filter)
+                return true if filter.nil?
+                return false if filter.equal?(FilterCompiler::NOTHING)
+
+                return filter["and"].all? { empty_match?(it) } if filter["and"]
+                return filter["or"].any? { empty_match?(it) } if filter["or"]
+
+                empty_test?(filter)
+            end
+
+            def empty_test?(value)
+                value.is_a?(Hash) && (value.key?("is_empty") ||
+                                      value.values.any? { empty_test?(it) })
+            end
+
+            # name => the conditions to send with that source's query
+            def split_where(sources)
+                conjuncts(@opts[:where]).group_by do |cond|
+                    owners = referenced(cond, sources)
+                    if owners.size > 1
+                        raise Error, "a join's where tests one table per " \
+                                     "condition: #{cond.inspect}"
+                    end
+
+                    owners.first || sources.first.first
+                end
+            end
+
+            def conjuncts(expr)
+                return [] if expr.nil?
+                if expr.is_a?(SQL::BooleanExpression) && expr.op == :AND
+                    return expr.args.flat_map { conjuncts(it) }
+                end
+
+                [expr]
+            end
+
+            def referenced(expr, sources)
+                case expr
+                when SQL::QualifiedIdentifier then [qualified(expr, sources)]
+                when Symbol, SQL::Identifier then [owner(expr, sources)]
+                when SQL::ComplexExpression then referenced(expr.args, sources)
+                when Array then expr.flat_map { referenced(it, sources) }.uniq
+                else []
+                end
+            end
+
+            def qualified(expr, sources)
+                name = expr.table.to_s
+                return name if sources.any? { it.first == name }
+
+                raise Error, "unknown table in a join: #{name}"
+            end
+
+            def owner(expr, sources)
+                column = FilterCompiler.property_name(expr).to_sym
+                found  = sources.select do |_, table|
+                    db.schema(table).any? { it.first == column }
+                end
+                return found.first.first if found.one?
+
+                raise Error, "column #{column} is in #{found.size} of the " \
+                             "joined tables; qualify it"
+            end
+        end
+    end
+end

data/lib/sequel/notion/model_support.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # Sequel::Model precomputes SQL for primary key lookups and
+        # deletes on "simple" tables, and runs it outside the dataset.
+        # A model over a Notion dataset is never simple, so those go
+        # through where(id: ...) like any other query.
+        module ModelSupport
+            # The model's dataset, allowing what Notion cannot compute to
+            # be computed in Ruby: Task.client_side.sum(:Hours)
+            def client_side(...) = dataset.client_side(...)
+
+            private
+
+            def convert_input_dataset(source)
+                ds = super
+                self.simple_table = nil if ds.is_a?(Notion::Dataset)
+                ds
+            end
+        end
+
+        # A row reads back partial values (a date's start only, the first
+        # 25 relations or people, rich text as plain text), so a save
+        # writes back only the columns that were changed, never the
+        # whole row.
+        module ModelSaveSupport
+            private
+
+            def _save_update_all_columns_hash
+                return super unless model.dataset.is_a?(Notion::Dataset)
+
+                _save_update_changed_columns_hash
+            end
+        end
+
+        # Notion cannot sort by page id: a model over a Notion dataset
+        # adds no primary key order, so paged_each streams in Notion's
+        # order and last needs an explicit one.
+        module ModelOrderSupport
+            private
+
+            def _primary_key_order
+                super unless is_a?(Notion::Dataset)
+            end
+        end
+
+        Sequel::Model::ClassMethods.prepend(ModelSupport)
+        Sequel::Model::DatasetMethods.prepend(ModelOrderSupport)
+        Sequel::Model::InstanceMethods.prepend(ModelSaveSupport)
+    end
+end