sequel-notion 0.2.0

data/lib/sequel/notion/dataset_having.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # HAVING over the groups DatasetGrouping computes, evaluated in
+        # Ruby with SQL's three-valued logic: a comparison with NULL is
+        # unknown, and only a true condition keeps a group. An aggregate
+        # it writes out (count(*) > 1) is computed as a hidden output.
+        module DatasetHaving
+            COMPARE = { "=": :==, "!=": :!=, "<": :<, ">": :>, "<=": :<=,
+                        ">=": :>= }.freeze
+
+            private
+
+            def having_functions(expr = @opts[:having], found = [])
+                case expr
+                when SQL::Function
+                    found << expr unless found.include?(expr)
+                when SQL::ComplexExpression, Array
+                    (expr.is_a?(Array) ? expr : expr.args)
+                        .each { having_functions(it, found) }
+                end
+                found
+            end
+
+            def having_name(index) = :"__having_#{index}"
+
+            def having?(row, funcs)
+                having_value(@opts[:having], row, funcs) == true
+            end
+
+            def having_value(expr, row, funcs)
+                case expr
+                when SQL::Function then row[having_name(funcs.index(expr))]
+                when SQL::BooleanExpression
+                    having_boolean(expr, row, funcs)
+                when Symbol, SQL::Identifier, SQL::QualifiedIdentifier
+                    having_column(expr, row)
+                when Array then expr.map { having_value(it, row, funcs) }
+                else having_literal(expr, row, funcs)
+                end
+            end
+
+            def having_literal(expr, row, funcs)
+                return expr unless expr.is_a?(SQL::Expression)
+                if expr.is_a?(SQL::ComplexExpression) && expr.op == :NOOP
+                    return having_value(expr.args.first, row, funcs)
+                end
+
+                raise Error, "Unsupported HAVING expression: #{expr.inspect}"
+            end
+
+            def having_column(expr, row)
+                name = group_column(expr)
+                return row[name] if row.key?(name)
+
+                raise Error,
+                      "HAVING names #{name}, neither grouped nor selected"
+            end
+
+            def having_boolean(expr, row, funcs)
+                values = expr.args.map { having_value(it, row, funcs) }
+                case expr.op
+                when :AND, :OR, :NOT then logical(expr.op, values)
+                when :IS, :"IS NOT" then identity(expr.op, *values)
+                when :IN, :"NOT IN" then in_list(expr.op, *values)
+                else compared(expr.op, *values)
+                end
+            end
+
+            def logical(op, values)
+                return (values.first.nil? ? nil : !values.first) if op == :NOT
+
+                decisive = op != :AND
+                return decisive if values.include?(decisive)
+
+                values.include?(nil) ? nil : !decisive
+            end
+
+            def identity(op, left, right)
+                same = right.nil? ? left.nil? : left == right
+                op == :IS ? same : !same
+            end
+
+            def in_list(op, left, list)
+                found = list.include?(left) unless left.nil?
+                found = nil if !found && (left.nil? || list.include?(nil))
+                op == :IN || found.nil? ? found : !found
+            end
+
+            def compared(op, left, right)
+                method = COMPARE.fetch(op) do
+                    raise Error, "Unsupported HAVING operator: #{op}"
+                end
+                left.nil? || right.nil? ? nil : left.public_send(method, right)
+            end
+        end
+    end
+end

data/lib/sequel/notion/dataset_joins.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require "sequel/notion/join_output"
+require "sequel/notion/join_where"
+require "sequel/notion/registry"
+
+module Sequel
+    module Notion
+        # INNER and LEFT joins on one equality, which Notion's API does
+        # not compute: each source is queried with the conditions that
+        # test it alone, then rows are matched in Ruby. A relation (an
+        # Array of page ids) matches every page it lists. Without a
+        # SELECT, a later table's column wins a shared name, as with SQL
+        # adapters.
+        module DatasetJoins
+            include JoinOutput
+            include JoinWhere
+
+            JOIN_TYPES = %i[inner left left_outer].freeze
+
+            private
+
+            def joined? = !@opts[:join].nil?
+
+            def join_rows(&)
+                sources = join_sources
+                rows    = joined_rows(sources).map { join_output(it, sources) }
+                rows    = sort_rows(rows).drop(@opts[:offset] || 0)
+                rows    = rows.take(@opts[:limit]) if @opts[:limit]
+                rows.each(&)
+            end
+
+            # { source name => row }, one per match
+            def joined_rows(sources)
+                wheres = split_where(sources)
+                first  = sources.first
+                rows   = source_rows(first, wheres)
+                         .map { { first.first => it } }
+                @opts[:join].each_with_index.reduce(rows) do |acc, (clause, i)|
+                    joined(acc, clause, sources[i + 1], wheres)
+                end
+            end
+
+            def join_sources
+                [[source_table.to_s, source_table]] +
+                    @opts[:join].map do |clause|
+                        table = clause.table
+                        table = table.value if table.is_a?(SQL::Identifier)
+                        [(clause.table_alias || table).to_s, table.to_sym]
+                    end
+            end
+
+            def source_rows((name, table), wheres)
+                ds = db[table]
+                Array(wheres[name]).each { ds = ds.where(it) }
+                ds.naked.all
+            end
+
+            def joined(rows, clause, (name, table), wheres)
+                left, right = join_columns(clause, name)
+                if clause.join_type != :inner && wheres[name]
+                    return left_joined(rows, left, right, [name, table],
+                                       wheres)
+                end
+
+                index = join_index(source_rows([name, table], wheres), right)
+                rows.flat_map { matched(it, left, index, clause, name) }
+            end
+
+            def join_index(rows, right)
+                index = Hash.new { |h, k| h[k] = [] }
+                rows.each do |row|
+                    join_keys(row[right]).each { index[it] << row }
+                end
+                index
+            end
+
+            def matched(row, (table, column), index, clause, name)
+                found = partners(row, table, column, index)
+                return found.map { row.merge(name => it) } unless found.empty?
+
+                clause.join_type == :inner ? [] : [row.merge(name => nil)]
+            end
+
+            def partners(row, table, column, index)
+                join_keys(row[table]&.[](column)).flat_map { index[it] }.uniq
+            end
+
+            # [[left source, column], right column] of an ON a = b
+            def join_columns(clause, name)
+                args        = join_on!(clause).args
+                right, left = args.partition { it.table.to_s == name }
+                unless right.size == 1
+                    raise Error, "a join's ON must test the joined table once"
+                end
+
+                [[left.first.table.to_s, left.first.column.to_sym],
+                 right.first.column.to_sym]
+            end
+
+            def join_on!(clause)
+                on = clause.on if clause.is_a?(SQL::JoinOnClause)
+                return on if JOIN_TYPES.include?(clause.join_type) &&
+                             equality?(on)
+
+                raise Error, "Unsupported join: #{clause.inspect}"
+            end
+
+            def equality?(on)
+                on.is_a?(SQL::BooleanExpression) && on.op == :"=" &&
+                    on.args.all?(SQL::QualifiedIdentifier)
+            end
+
+            # A value's join keys: a relation's ids, page ids without
+            # dashes and case
+            def join_keys(value)
+                Array(value).map do |v|
+                    uuid = v.is_a?(String) && v.match?(Registry::UUID)
+                    uuid ? v.delete("-").downcase : v
+                end
+            end
+        end
+    end
+end

data/lib/sequel/notion/dataset_pages.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require "sequel/notion/errors"
+require "sequel/notion/filter_compiler"
+require "sequel/notion/sort_compiler"
+
+module Sequel
+    module Notion
+        # Where a dataset's pages come from: a paginated data source
+        # query, or a GET per page when WHERE pins the page id.
+        module DatasetPages
+            # Clauses Notion cannot express; refused rather than ignored
+            UNSUPPORTED = %i[join group having compounds lock].freeze
+
+            PAGE_SIZE = 100
+
+            private
+
+            def page_ids
+                ids = []
+                each_notion_page { ids << it["id"] }
+                ids
+            end
+
+            # Yield raw pages, after OFFSET and up to LIMIT
+            def each_notion_page
+                check_supported!
+                skip = @opts[:offset] || 0
+                max  = @opts[:limit]
+                seen = 0
+                source_pages do |page|
+                    next if (skip -= 1) >= 0
+
+                    yield page
+                    break if max && (seen += 1) >= max
+                end
+            end
+
+            def check_supported!
+                bad = UNSUPPORTED.select { @opts[it] }
+                return if bad.empty?
+
+                raise Error, "Notion datasets do not support: " \
+                             "#{bad.join(", ")}"
+            end
+
+            def source_pages(&)
+                ds_id = data_source_id
+                ids, rest = split_id_condition(@opts[:where])
+                return pages_by_id(ds_id, ids, rest, &) if ids
+
+                filter = @opts[:where] &&
+                         FilterCompiler.for(db, ds_id).compile(@opts[:where])
+                query_pages(ds_id, filter, &) unless
+                    filter.equal?(FilterCompiler::NOTHING)
+            end
+
+            def query_pages(ds_id, filter)
+                cursor = nil
+                loop do
+                    resp = db.notion_query(ds_id, query_body(filter, cursor))
+                    (resp["results"] || []).each { yield it }
+                    break unless resp["has_more"]
+
+                    cursor = resp["next_cursor"]
+                end
+            end
+
+            def query_body(filter, cursor)
+                body = { page_size: notion_page_size }
+                body[:filter] = filter if filter
+                body[:sorts]  = SortCompiler.compile(@opts[:order]) if
+                    @opts[:order]
+                body[:start_cursor] = cursor if cursor
+                body
+            end
+
+            def notion_page_size
+                offset = @opts[:offset] || 0
+                want   = @opts[:limit] && (@opts[:limit] + offset)
+                [want || PAGE_SIZE, @opts[:notion_page_size] || PAGE_SIZE,
+                 PAGE_SIZE].min
+            end
+
+            # ----------------------------------------------------------
+            # Lookup by page id
+            # ----------------------------------------------------------
+
+            # [ids, other conditions] when WHERE pins the id, else nil
+            def split_id_condition(where)
+                return unless where.is_a?(SQL::BooleanExpression)
+                return id_and_rest(where.args) if where.op == :AND
+
+                ids = id_values(where)
+                [ids, []] if ids
+            end
+
+            def id_and_rest(conds)
+                sets = conds.filter_map { id_values(it) }
+                return if sets.empty?
+
+                [sets.reduce(:&), conds.reject { id_values(it) }]
+            end
+
+            def id_values(expr)
+                return unless expr.is_a?(SQL::BooleanExpression)
+                return unless %i[= IN].include?(expr.op)
+                return unless FilterCompiler.property_name(expr.args[0]) ==
+                              "id"
+
+                Array(expr.args[1]).map { bare_id(it) }.uniq
+            rescue Error
+                nil
+            end
+
+            def pages_by_id(ds_id, ids, rest)
+                unless rest.empty?
+                    raise Error, "an id lookup cannot be combined with " \
+                                 "other conditions"
+                end
+
+                pages = ids.lazy.filter_map { fetch_page(it) }
+                           .select { in_source?(it, ds_id) }
+                pages = ordered_pages(pages.to_a) if @opts[:order]
+                pages.each { yield it }
+            end
+
+            # Trashed pages included: the row says so, and this is the
+            # only way to reach one to restore it
+            def fetch_page(id)
+                db.notion_get_page(id)
+            rescue NotFoundError
+                nil
+            end
+
+            def in_source?(page, ds_id)
+                parent = page["parent"] || {}
+                parent["type"] == "data_source_id" &&
+                    bare_id(parent["data_source_id"]) == bare_id(ds_id)
+            end
+
+            def bare_id(id) = id.to_s.delete("-").downcase
+        end
+    end
+end

data/lib/sequel/notion/dataset_selection.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "sequel/notion/filter_compiler"
+
+module Sequel
+    module Notion
+        # SELECT for Notion datasets: plain columns, optionally aliased
+        module DatasetSelection
+            private
+
+            # [[source column, output name], ...], or nil for every column
+            def selection
+                sel = @opts[:select]
+                return if sel.nil? || sel.empty?
+                return if sel.any? { it == :* || it.is_a?(SQL::ColumnAll) }
+
+                known!(sel.map { selected(it) })
+            end
+
+            def known!(pairs)
+                unknown = pairs.map(&:first) -
+                          db.schema(source_table).map(&:first)
+                return pairs if unknown.empty?
+
+                raise Error, "Unknown columns in select: #{unknown.join(", ")}"
+            end
+
+            def selected(expr)
+                if expr.is_a?(SQL::AliasedExpression)
+                    [column_name(expr.expression), expr.alias.to_sym]
+                else
+                    name = column_name(expr)
+                    [name, name]
+                end
+            end
+
+            def column_name(expr)
+                FilterCompiler.property_name(expr).to_sym
+            rescue Error
+                raise Error, "Unsupported select expression: " \
+                             "#{expr.inspect} (only columns can be selected)"
+            end
+
+            def project(row, sel)
+                return row unless sel
+
+                sel.to_h { |source, name| [name, row[source]] }
+            end
+        end
+    end
+end

data/lib/sequel/notion/dataset_truncation.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # A page lists at most 25 relations or people; the rest is read
+        # from the page property endpoint, for the properties a row
+        # returns. A relation says has_more; people say nothing, so 25
+        # of them may be more.
+        module DatasetTruncation
+            LISTED = 25
+
+            private
+
+            def complete_page(page, sel)
+                wanted = sel&.map { it.first.to_s }
+                (page["properties"] || {}).each do |name, prop|
+                    next if wanted && !wanted.include?(name)
+
+                    complete_property(page["id"], prop) if truncated?(prop)
+                end
+                page
+            end
+
+            def complete_property(page_id, prop)
+                type = prop["type"]
+                prop[type] = db.notion_property_items(page_id, prop["id"])
+                               .map { it[type] }
+            end
+
+            def truncated?(prop)
+                case prop["type"]
+                when "relation" then prop["has_more"] == true
+                when "people" then Array(prop["people"]).size >= LISTED
+                else false
+                end
+            end
+        end
+    end
+end

data/lib/sequel/notion/discovery.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # Listing the data sources the integration can see
+        module Discovery
+            # List data sources: those of one database, or a search
+            def data_sources(database: nil, query: nil)
+                if database && query
+                    raise ArgumentError, "database and query are exclusive"
+                end
+
+                database ? database_sources(database) : search_sources(query)
+            end
+
+            private
+
+            # Notion's search still lists a trashed data source, flagged
+            def live(sources) = sources.reject { it[:in_trash] }
+
+            # Ids of the live data sources whose title normalises to name
+            def matching_source_ids(name)
+                wanted = Registry.normalize(name)
+                return [] if wanted.empty?
+
+                live(search_sources(name.to_s.tr("_", " ")))
+                    .select { Registry.normalize(it[:name]) == wanted }
+                    .map { it[:id] }.uniq
+            end
+
+            def database_sources(database)
+                resp = request(:get, "databases/#{database}")
+                (resp["data_sources"] || []).map do |ds|
+                    ds.slice("id", "name").transform_keys(&:to_sym)
+                      .merge(parent_database_id: database)
+                end
+            end
+
+            def search_sources(query)
+                sources = []
+                cursor  = nil
+                loop do
+                    resp = request(:post, "search", search_body(query, cursor))
+                    sources.concat(source_entries(resp["results"]))
+                    break unless resp["has_more"]
+
+                    cursor = resp["next_cursor"]
+                end
+                sources
+            end
+
+            def source_entries(results)
+                (results || []).filter_map do |entry|
+                    source_entry(entry) if entry["object"] == "data_source"
+                end
+            end
+
+            def search_body(query, cursor)
+                body = { page_size: 100,
+                         filter: { property: "object", value: "data_source" },
+                         sort: { direction: "descending",
+                                 timestamp: "last_edited_time" } }
+                body[:query]        = query  if query
+                body[:start_cursor] = cursor if cursor
+                body
+            end
+
+            def source_entry(ds)
+                ds.slice("id", "icon", "url", "in_trash", "properties")
+                  .transform_keys(&:to_sym)
+                  .merge(parent_database_id: ds.dig("parent", "database_id"),
+                         name: ds["title"]&.map { it["plain_text"] }&.join)
+            end
+        end
+    end
+end

data/lib/sequel/notion/errors.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # A Notion object (page, data source) that does not exist, or that
+        # the integration cannot see
+        class NotFoundError < Sequel::DatabaseError; end
+    end
+end

data/lib/sequel/notion/filter_comparison.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require "date"
+require "sequel/notion/filter_tables"
+
+module Sequel
+    module Notion
+        # Standard and date-inequality (rule D) comparison compilation,
+        # mixed into FilterCompiler; formulas and rollups are FilterNested's.
+        # All private.
+        module FilterComparison
+            private
+
+            # ----------------------------------------------------------
+            # Standard comparison
+            # ----------------------------------------------------------
+
+            def compile_comparison(expr)
+                case expr.op
+                when :IN, :"NOT IN" then return compile_in(expr)
+                when :LIKE, :ILIKE, :"NOT LIKE", :"NOT ILIKE"
+                    return compile_like(expr)
+                end
+
+                name, op, value = comparison_operands(expr)
+                return null_comparison_filter(name, op) if value.nil?
+
+                type = lookup_type!(name)
+                key  = filter_key_for!(name, type)
+
+                compile_typed_comparison(name, op, value, key)
+            end
+
+            # Swaps a comparison's operands into column-then-value order
+            # (rule S): a < b  ==  b > a
+            def comparison_operands(expr)
+                op = expr.op
+                left, right = expr.args
+
+                if !column_expr?(left) && column_expr?(right)
+                    left, right = right, left
+                    op = mirrored_operator(op)
+                end
+
+                [FilterCompiler.property_name(left), op,
+                 literal_value!(unwrap_literal(right))]
+            end
+
+            # Notion compares a property with a value, never with another
+            # property or an expression
+            def literal_value!(value)
+                return value unless bare_column_expr?(value) ||
+                                    value.is_a?(Sequel::SQL::Expression)
+
+                raise Sequel::Error,
+                      "Notion filters compare with values only: " \
+                      "#{value.inspect}"
+            end
+
+            def mirrored_operator(op)
+                FilterTables::MIRROR_OPERATOR.fetch(op) do
+                    raise Sequel::Error, "Unsupported operator: #{op}"
+                end
+            end
+
+            def null_comparison_filter(name, op)
+                case op
+                when :"=" then null_check_filter(name, empty: true)
+                when :"!=" then null_check_filter(name, empty: false)
+                else
+                    raise Sequel::Error,
+                          "Unsupported nil comparison operator: #{op}"
+                end
+            end
+
+            def compile_typed_comparison(name, op, value, key)
+                if FilterNested::NESTED_KEYS.include?(key)
+                    return compile_nested_key(name, op, value, key)
+                end
+                if key == "date" && op == :"!="
+                    return date_not_equal_filter(name, value)
+                end
+
+                notion_op = translated_operator(op, key)
+                ensure_supported!(name, key, notion_op)
+
+                { "property" => name,
+                  key => { notion_op => coerce_value(value, key) } }
+            end
+
+            # Looks up the Notion operator for `op`, remapping it for date
+            # properties and for the types that express (not)equal through
+            # contains/does_not_contain (rule IN).
+            def translated_operator(op, key)
+                notion_op = FilterTables::OPERATOR_MAP.fetch(op) do
+                    raise Sequel::Error, "Unsupported operator: #{op}"
+                end
+                if key == "date"
+                    notion_op = FilterTables::DATE_OPERATOR_MAP.fetch(
+                        notion_op, notion_op
+                    )
+                end
+                contains_type?(key) ? equal_to_contains(notion_op) : notion_op
+            end
+
+            # ----------------------------------------------------------
+            # Date inequality (rule D)
+            # ----------------------------------------------------------
+
+            def date_not_equal_filter(name, value)
+                coerced = coerce_value(value, "date")
+                {
+                    "or" => [
+                        { "property" => name,
+                          "date" => { "before" => coerced } },
+                        { "property" => name,
+                          "date" => { "after" => coerced } }
+                    ]
+                }
+            end
+        end
+    end
+end