sequel-notion 0.2.0

data/lib/sequel/notion/notion_file.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Sequel
+    module Notion
+        # File's "Notion API → File instances" class methods, split out
+        # here (and mixed in with `extend`) to keep File's own body
+        # under the class-length limit.
+        module FileDeserialization
+            # Parse a single file object from Notion API response
+            #
+            #   File.from_notion({
+            #     "type" => "external",
+            #     "name" => "doc.pdf",
+            #     "external" => { "url" => "https://..." }
+            #   })
+            #
+            def from_notion(file_obj)
+                return nil unless file_obj.is_a?(Hash)
+
+                case file_obj["type"]
+                when "file"     then from_notion_file(file_obj)
+                when "external" then from_notion_external(file_obj)
+                when nil then raise Sequel::Error, "file object with no type"
+                else from_notion_other(file_obj)
+                end
+            end
+
+            # Parse the "files" property value → array of File
+            #
+            #   File.from_notion_property(prop)
+            #   # => [#<File name="doc.pdf" url="https://...">, ...]
+            #
+            def from_notion_property(prop)
+                files = prop["files"] || []
+                files.filter_map { |f| from_notion(f) }
+            end
+
+            private
+
+            def extract_caption(file_obj)
+                file_obj["caption"]&.map { |t| t["plain_text"] }&.join
+            end
+
+            # from_notion's three branches, split out to keep from_notion
+            # itself short.
+
+            def from_notion_file(file_obj)
+                new(
+                    name: file_obj["name"],
+                    url: file_obj.dig("file", "url"),
+                    type: :file,
+                    expiry_time: file_obj.dig("file", "expiry_time"),
+                    caption: extract_caption(file_obj)
+                )
+            end
+
+            def from_notion_external(file_obj)
+                new(
+                    name: file_obj["name"],
+                    url: file_obj.dig("external", "url"),
+                    type: :external,
+                    caption: extract_caption(file_obj)
+                )
+            end
+
+            # A type this adapter does not know yet (Notion added one
+            # after this was written): round-trip it opaquely via @raw
+            def from_notion_other(file_obj)
+                type = file_obj["type"]
+                new(
+                    name: file_obj["name"],
+                    url: file_obj.dig(type, "url"),
+                    type: type,
+                    caption: extract_caption(file_obj),
+                    raw: file_obj
+                )
+            end
+        end
+
+        class File
+            extend FileDeserialization
+
+            attr_reader :name, :url, :expiry_time, :type, :caption, :raw
+
+            # Keywords besides url:, name: and type:
+            OPTIONS = %i[expiry_time caption raw].freeze
+
+            # type: :file (Notion-hosted) or :external; any other is kept
+            # with its raw hash, so that it is written back unchanged
+            def initialize(url:, name: nil, type: :external, **opts)
+                unknown = opts.keys - OPTIONS
+                unless unknown.empty?
+                    raise ArgumentError, "unknown keywords: #{unknown.inspect}"
+                end
+
+                @name        = name
+                @url         = url
+                @type        = type.to_sym
+                @expiry_time = opts[:expiry_time]
+                @caption     = opts[:caption]
+                @raw         = opts[:raw]
+            end
+
+            # Is this a Notion-hosted (internal) file?
+            def internal? = @type == :file
+
+            # Is this an external URL?
+            def external? = @type == :external
+
+            # Has the Notion-hosted URL expired?
+            def expired?
+                return false unless internal? && @expiry_time
+
+                Time.parse(@expiry_time) < Time.now
+            end
+
+            # ----------------------------------------------------------
+            # Serialisation: File → Notion API property value
+            # ----------------------------------------------------------
+
+            # Single file → Notion file object (for use inside arrays)
+            def to_notion
+                case @type
+                when :external then to_notion_external
+                when :file then to_notion_file
+                else @raw
+                end
+            end
+
+            # Build Notion "files" property value from one or more Files
+            #
+            #   File.to_notion_property([file1, file2])
+            #   # => { "files" => [ { ... }, { ... } ] }
+            #
+            def self.to_notion_property(files)
+                files = Array(files)
+                { "files" => files.map(&:to_notion) }
+            end
+
+            # ----------------------------------------------------------
+            # Convenience constructors
+            # ----------------------------------------------------------
+
+            # Create an external file reference
+            #
+            #   File.external("https://example.com/doc.pdf", name: "My Doc")
+            #
+            def self.external(url, name: nil)
+                new(url: url, type: :external, name: name)
+            end
+
+            # Wrap multiple URLs into File instances
+            #
+            #   File.from_urls("https://a.com/1.pdf", "https://b.com/2.pdf")
+            #
+            def self.from_urls(*urls)
+                urls.flatten.map { |u| external(u) }
+            end
+
+            # ----------------------------------------------------------
+            # Comparison & display
+            # ----------------------------------------------------------
+
+            # raw tells apart files of an unknown type, which may have no URL
+            def ==(other)
+                other.is_a?(File) && @url == other.url &&
+                    @type == other.type && @raw == other.raw
+            end
+            alias eql? ==
+
+            def hash
+                [@url, @type, @raw].hash
+            end
+
+            def to_s
+                tag = expired? ? " [EXPIRED]" : ""
+                "#<#{self.class}:#{@type} #{@name || "unnamed"} → " \
+                    "#{@url}#{tag}>"
+            end
+            alias inspect to_s
+
+            private
+
+            # to_notion's two branches, split out to keep to_notion itself
+            # short.
+
+            def to_notion_external
+                uri = parsed_url
+                { "type" => "external", "name" => @name || default_name(uri),
+                  "external" => { "url" => @url } }
+            end
+
+            def parsed_url
+                URI.parse(@url)
+            rescue URI::InvalidURIError
+                raise Sequel::Error, "invalid file URL: #{@url.inspect}"
+            end
+
+            # The URL's last path segment, decoded ("a%20b.pdf" is
+            # "a b.pdf"; one that does not decode stays as it is), or the
+            # URL when it has none
+            def default_name(uri)
+                base = ::File.basename(uri.path.to_s)
+                return @url if ["", "/"].include?(base)
+
+                name = URI.decode_uri_component(base)
+                name.valid_encoding? ? name : base
+            rescue ArgumentError
+                base
+            end
+
+            def to_notion_file
+                # A Notion-hosted file is written back as read, signed URL
+                # and all, which Notion accepts and keeps the file for
+                # (checked live); a new one is uploaded with file_upload.
+                obj = {
+                    "type" => "file",
+                    "name" => @name || "",
+                    "file" => { "url" => @url }
+                }
+                obj["file"]["expiry_time"] = @expiry_time if @expiry_time
+                obj
+            end
+        end
+    end
+end

data/lib/sequel/notion/page_api.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # The page and query endpoints the Dataset calls, over
+        # Database#request
+        module PageApi
+            def notion_query(data_source_id, body)
+                request(:post, "data_sources/#{data_source_id}/query", body)
+            end
+
+            def notion_get_page(page_id)
+                request(:get, "pages/#{page_id}")
+            end
+
+            # Every item of a paginated page property (relation, people)
+            def notion_property_items(page_id, property_id)
+                path   = "pages/#{page_id}/properties/#{property_id}"
+                items  = []
+                cursor = nil
+                loop do
+                    resp = request(:get, path,
+                                   cursor && { start_cursor: cursor })
+                    items.concat(resp["results"] || [])
+                    break items unless resp["has_more"]
+
+                    cursor = resp["next_cursor"]
+                end
+            end
+
+            def notion_create_page(data_source_id, properties)
+                request(:post, "pages",
+                        parent: { type: "data_source_id",
+                                  data_source_id: data_source_id },
+                        properties: properties)
+            end
+
+            def notion_update_page(page_id, properties, in_trash: nil)
+                body = { properties: properties }
+                body[:in_trash] = in_trash unless in_trash.nil?
+                request(:patch, "pages/#{page_id}", body)
+            end
+
+            def notion_trash_page(page_id)
+                request(:patch, "pages/#{page_id}", in_trash: true)
+            end
+        end
+    end
+end

data/lib/sequel/notion/registry.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "sequel/notion/discovery"
+
+module Sequel
+    module Notion
+        # Table name => data source id, filled explicitly, by discovery, or
+        # lazily by searching for a data source of that name.
+        module Registry
+            include Discovery
+
+            UUID = /\A\h{8}-?\h{4}-?\h{4}-?\h{4}-?\h{12}\z/
+
+            # A discovered name that several data sources share: looking
+            # it up raises, until a registration picks one
+            Ambiguous = Data.define(:ids)
+
+            # The table name a data source title maps to: accents dropped
+            # from Latin letters only, letters of other scripts kept
+            def self.normalize(title)
+                title.to_s.unicode_normalize(:nfkd)
+                     .gsub(/(?<=\p{Latin})\p{Mn}+/, "")
+                     .unicode_normalize(:nfc)
+                     .downcase
+                     .gsub(/[^\p{L}\p{N}]+/, "_")
+                     .gsub(/\A_|_\z/, "")
+            end
+
+            def register_data_source(name, data_source_id)
+                raise Error, "Missing data source id for #{name}" unless
+                    data_source_id
+
+                # Notion answers 404 to an upper-case id
+                registry_store(name.to_sym, data_source_id.to_s.downcase)
+                self
+            end
+
+            # Register every data source found (in one database, or by a
+            # search); the block, given (title, id), may choose the name.
+            # All or nothing: a name clash registers none of them.
+            def register_all_data_sources(database: nil, query: nil, &mapper)
+                found = live(data_sources(database:, query:)).map do |ds|
+                    [source_name(ds, mapper), ds[:id]]
+                end
+                Sequel.synchronize do
+                    staged = registry.dup
+                    found.each { |name, id| bind(staged, name, id) }
+                    registry.replace(staged)
+                end
+                self
+            end
+
+            def data_source_id_for(table_name)
+                name = table_name.to_sym
+                resolved(name, registry_fetch(name)) ||
+                    (name.match?(UUID) && name.to_s.downcase) ||
+                    (auto_register? && resolved(name, registry_fetch(name))) ||
+                    search_data_source(name)
+            end
+
+            def tables
+                auto_register?
+                Sequel.synchronize do
+                    registry.reject { |_, id| id.is_a?(Ambiguous) }.keys
+                end
+            end
+
+            private
+
+            def registry = (@registry ||= {})
+
+            # A title with no letter or digit is named by its id
+            def source_name(source, mapper)
+                return mapper.call(source[:name], source[:id]).to_sym if mapper
+
+                name = Registry.normalize(source[:name])
+                (name.empty? ? source[:id] : name).to_sym
+            end
+
+            def registry_fetch(name) = Sequel.synchronize { registry[name] }
+
+            def registry_store(name, id)
+                Sequel.synchronize { bind(registry, name, id) }
+            end
+
+            def resolved(name, id)
+                return id unless id.is_a?(Ambiguous)
+
+                raise Error, "data source name #{name} is ambiguous " \
+                             "(#{id.ids.join(", ")}): register one"
+            end
+
+            # A registration may resolve an ambiguous name, never rebind
+            # one already bound
+            def bind(map, name, id)
+                old = map[name]
+                if old && old != id && !old.is_a?(Ambiguous)
+                    raise Error, "data source name #{name} already maps " \
+                                 "to #{old}, not #{id}"
+                end
+                map[name] = id
+            end
+
+            # Discover every data source once, when auto_register is set;
+            # true when it is set. A name two sources share is marked
+            # ambiguous rather than failing the others, and a name already
+            # registered is kept. As Sequel does for its schema cache, the
+            # flag is set only once discovery has succeeded, and no lock is
+            # held across the requests: a concurrent lookup repeats the
+            # discovery rather than read a registry half filled, and a
+            # failed one is retried on the next lookup.
+            def auto_register?
+                return false unless opts[:auto_register]
+
+                unless Sequel.synchronize { @auto_registered }
+                    discover_data_sources
+                    Sequel.synchronize { @auto_registered = true }
+                end
+                true
+            end
+
+            def discover_data_sources
+                found = live(data_sources).group_by { source_name(it, nil) }
+                Sequel.synchronize do
+                    found.each { |name, sources| discovered(name, sources) }
+                end
+            end
+
+            def discovered(name, sources)
+                return if registry[name] in String
+
+                ids = sources.map { it[:id] }.uniq
+                registry[name] = ids.one? ? ids.first : Ambiguous[ids]
+            end
+
+            # Fallback: search by name, and remember what was found
+            def search_data_source(name)
+                ids = matching_source_ids(name)
+                return if ids.empty?
+                if ids.size > 1
+                    raise Error, "data source name #{name} is ambiguous: " \
+                                 "#{ids.join(", ")}"
+                end
+
+                registry_store(name, ids.first)
+                ids.first
+            end
+        end
+    end
+end

data/lib/sequel/notion/request_budget.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Sequel
+    module Notion
+        # A ceiling on the Notion requests one query makes, mixed into
+        # Database. The budget covers every request answered 200 while
+        # the query runs, in this fiber, including the queries a join or
+        # a union runs inside it; the request past it raises before it is
+        # sent. A rate-limited attempt the retry absorbs, or a request
+        # that fails, spends nothing.
+        module RequestBudget
+            # Run the block under a budget of +max+ requests (nil: none).
+            # A query run inside a budgeted one spends the outer budget.
+            def with_request_budget(max)
+                return yield if max.nil? || Thread.current[budget_key]
+
+                begin
+                    Thread.current[budget_key] = [max, 0]
+                    yield
+                ensure
+                    Thread.current[budget_key] = nil
+                end
+            end
+
+            # Run the block with no budget: what the caller does with a
+            # row (another query, say) is no part of the query that
+            # gave it
+            def outside_request_budget
+                saved = Thread.current[budget_key]
+                Thread.current[budget_key] = nil
+                yield
+            ensure
+                Thread.current[budget_key] = saved
+            end
+
+            private
+
+            # The block's answer, refused once the budget is spent and
+            # counted once it succeeds
+            def within_budget
+                check_request_budget!
+                yield.tap { spend_request! }
+            end
+
+            def budget_key = :"notion_budget_#{object_id}"
+
+            # Called before each request: refuse it once the budget is spent
+            def check_request_budget!
+                budget = Thread.current[budget_key] or return
+                return if budget.last < budget.first
+
+                raise Error, "query stopped after #{budget.first} Notion " \
+                             "request(s), its max_requests; raise it in " \
+                             "client_side(max_requests:)"
+            end
+
+            # Called after each request answered 200
+            def spend_request!
+                budget = Thread.current[budget_key] or return
+                budget[1] += 1
+            end
+        end
+    end
+end

data/lib/sequel/notion/schema.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "sequel/notion/type_map"
+
+module Sequel
+    module Notion
+        module Schema
+            NOTION_TO_SEQUEL = {
+                "title" => :string,
+                "rich_text" => :string,
+                "number" => :float,
+                "select" => :string,
+                "multi_select" => :array,
+                "status" => :string,
+                "date" => :notion_date, # no Sequel typecast: kept as given
+                "checkbox" => :boolean,
+                "url" => :string,
+                "email" => :string,
+                "phone_number" => :string,
+                "people" => :array,
+                "relation" => :array,
+                "files" => :array,
+                "formula" => :string,
+                "created_time" => :string,
+                "last_edited_time" => :string
+            }.freeze
+
+            # Columns every row carries, besides the data source properties
+            PAGE_COLUMNS = [
+                [:id,       { type: :string,  db_type: "page_id",
+                              primary_key: true, allow_null: false }],
+                [:in_trash, { type: :boolean, db_type: "in_trash",
+                              allow_null: false }]
+            ].freeze
+
+            # Rollup functions by the kind of value they give; any other
+            # (date_range, a new one) is unknown and not filterable
+            ROLLUP_KINDS = {
+                "array" => %w[show_original show_unique],
+                "date" => %w[earliest_date latest_date],
+                "number" => %w[count count_values empty not_empty unique
+                               percent_empty percent_not_empty sum average
+                               median min max range checked unchecked
+                               percent_checked percent_unchecked
+                               count_per_group percent_per_group]
+            }.freeze
+
+            module_function
+
+            def rollup_kind(prop)
+                function = prop.dig("rollup", "function")
+                ROLLUP_KINDS.find { |_, fns| fns.include?(function) }&.first
+            end
+
+            # Data source properties => Sequel schema rows
+            def notion_to_sequel(properties)
+                PAGE_COLUMNS.map { |name, info| [name, info.dup] } +
+                    properties.map do |name, prop|
+                        [name.to_sym, column(prop["type"])]
+                    end
+            end
+
+            def column(notion_type)
+                { type: NOTION_TO_SEQUEL.fetch(notion_type, :string),
+                  generated: TypeMap::READ_ONLY_TYPES.include?(notion_type),
+                  db_type: notion_type,
+                  notion_type: notion_type,
+                  allow_null: true,
+                  default: nil,
+                  primary_key: false }
+            end
+        end
+    end
+end

data/lib/sequel/notion/schema_lookup.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "sequel/notion/schema"
+
+module Sequel
+    module Notion
+        # A data source's properties, fetched once and cached, and what
+        # Sequel (columns) and the filter compiler (types, rollup kinds)
+        # read from them; mixed into Database
+        module SchemaLookup
+            # Sequel caches columns per table name, and another name for
+            # the data source (an alias, its id) would keep the old ones:
+            # all are dropped, rebuilt from the data source cache with no
+            # request
+            def refresh_schema!(table_name)
+                ds_id = data_source_id_for(table_name)
+                Sequel.synchronize do
+                    @data_source_cache.delete(ds_id)
+                    @data_source_epoch[ds_id] += 1
+                    @schemas.clear
+                end
+                remove_cached_schema(table_name)
+            end
+
+            # Property name => Notion type, for one data source
+            def property_type_map(ds_id)
+                data_source(ds_id).transform_values { it["type"] }
+            end
+
+            # Rollup name => the kind of value it gives, for one data source
+            def rollup_kinds(ds_id)
+                data_source(ds_id).select { |_, p| p["type"] == "rollup" }
+                                  .transform_values { Schema.rollup_kind(it) }
+            end
+
+            private
+
+            # Data source object properties, fetched once and cached. A
+            # fetch that a refresh_schema! overtook fetches again, so its
+            # older answer never replaces the newer one
+            def data_source(ds_id)
+                loop do
+                    epoch, cached = cached_properties(ds_id)
+                    return cached if cached
+
+                    props = fetch_properties(ds_id)
+                    return props if store_properties(ds_id, epoch, props)
+                end
+            end
+
+            def cached_properties(ds_id)
+                Sequel.synchronize do
+                    [@data_source_epoch[ds_id], @data_source_cache[ds_id]]
+                end
+            end
+
+            # Cache them, unless a refresh came in meanwhile
+            def store_properties(ds_id, epoch, props)
+                Sequel.synchronize do
+                    next false unless @data_source_epoch[ds_id] == epoch
+
+                    @data_source_cache[ds_id] = props
+                    true
+                end
+            end
+
+            def fetch_properties(ds_id)
+                props = request(:get,
+                                "data_sources/#{ds_id}")["properties"] || {}
+                TypeMap.check_property_names!(props.keys)
+                props
+            end
+
+            def schema_parse_table(table_name, _opts)
+                ds_id = data_source_id_for(table_name)
+                raise Error, "Unknown data source: #{table_name}" unless ds_id
+
+                Schema.notion_to_sequel(data_source(ds_id))
+            end
+        end
+    end
+end