notion_publish 0.1.0

data/lib/notion_publish/manifest.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require "monitor"
+require "pathname"
+require "yaml"
+
+require_relative "errors"
+
+module NotionPublish
+  # Where each document was published, and enough about it to tell whether
+  # republishing has anything to do.
+  #
+  # Tool-written, never hand-edited, committed alongside the documents. One per
+  # repository so that links between documents in different directories can be
+  # resolved from a single place.
+  class Manifest
+    FILENAME = "notion-publish-manifest.yml"
+
+    HEADER = <<~TEXT
+      # Generated by notion-publish. Do not edit by hand.
+      # Maps each Markdown source to the Notion page it mirrors.
+      # To re-point an entry, delete it and publish again.
+      # Settings belong in .notion-publish.yml, which this tool never rewrites.
+    TEXT
+
+    ENTRY_FIELDS = %w[id url parent properties flag_properties title_override keep_h1 source_sha256
+                      properties_sha256 flag_properties_sha256 notion_sha256 published_at].freeze
+
+    # One published document.
+    #
+    # +flag_properties+ names the properties whose values came from
+    # --property or --properties-json rather than front matter. Their values
+    # are not recorded, so `republish` leaves them alone. It is nil on entries
+    # written before it existed, which is how `republish` recognises them.
+    # +title_override+ and +keep_h1+ record --title and --keep-h1 for the
+    # same reason.
+    Entry = Data.define(:id, :url, :parent, :properties, :flag_properties, :title_override, :keep_h1,
+                        :source_sha256, :properties_sha256, :flag_properties_sha256,
+                        :notion_sha256, :published_at) do
+      # Only the page itself is required. Everything else accumulates: adopting
+      # records no source hash, and older entries predate later fields.
+      def initialize(id:, url:, parent: nil, properties: [], flag_properties: nil, title_override: nil,
+                     keep_h1: nil, source_sha256: nil, properties_sha256: nil, flag_properties_sha256: nil,
+                     notion_sha256: nil, published_at: nil)
+        super
+      end
+
+      def to_h
+        ENTRY_FIELDS.to_h { |field| [field, public_send(field)] }.compact
+      end
+
+      def self.from(hash)
+        new(**ENTRY_FIELDS.to_h { |field| [field.to_sym, hash[field]] }, properties: hash["properties"] || [])
+      end
+    end
+
+    attr_reader :path
+
+    # --manifest wins; then an existing manifest walking up from the
+    # document; then the repository root; then the document's own directory.
+    def self.locate(document_path, override: nil)
+      locate_in(File.dirname(File.expand_path(document_path)), override: override)
+    end
+
+    # The same search, starting from a directory rather than a document.
+    def self.locate_in(dir, override: nil)
+      return new(File.expand_path(override)) if override
+
+      found = walk_up(dir) { |d| File.file?(File.join(d, FILENAME)) }
+      return new(File.join(found, FILENAME)) if found
+
+      root = walk_up(dir) { |d| File.directory?(File.join(d, ".git")) }
+      new(File.join(root || dir, FILENAME))
+    end
+
+    def self.walk_up(dir)
+      loop do
+        return dir if yield(dir)
+
+        parent = File.dirname(dir)
+        return nil if parent == dir
+
+        dir = parent
+      end
+    end
+
+    def initialize(path)
+      @path = path
+      @data = File.file?(path) ? load_file : {}
+      @created = !File.file?(path)
+      # Keys removed in memory, so the merge with disk does not put them back.
+      @forgotten = []
+      # Several pages are checked and recorded at once; the file lock guards
+      # other processes, this guards the threads of this one.
+      @lock = Monitor.new
+    end
+
+    def dir = File.dirname(path)
+    def created? = @created
+    def pages = @data["pages"] ||= {}
+    def workspace_id = @data["workspace_id"]
+
+    def workspace_id=(value)
+      @lock.synchronize do
+        @data["workspace_id"] = value if value && @data["workspace_id"] != value
+      end
+    end
+
+    def key_for(absolute_path)
+      Pathname.new(File.expand_path(absolute_path)).relative_path_from(Pathname.new(dir)).to_s
+    rescue ArgumentError
+      File.expand_path(absolute_path)
+    end
+
+    def entry(absolute_path)
+      @lock.synchronize do
+        raw = pages[key_for(absolute_path)]
+        raw && Entry.from(raw)
+      end
+    end
+
+    def url_for(absolute_path) = entry(absolute_path)&.url
+
+    # Recording a key that was forgotten earlier in this run takes it off the
+    # forgotten list, or the save would delete the entry it is writing.
+    def record(absolute_path, entry)
+      @lock.synchronize do
+        key = key_for(absolute_path)
+        @forgotten.delete(key)
+        pages[key] = entry.to_h
+        save
+      end
+    end
+
+    def forget(absolute_path)
+      @lock.synchronize do
+        key = key_for(absolute_path)
+        pages.delete(key)
+        @forgotten << key
+        save
+      end
+    end
+
+    # Entries whose source file no longer exists. Reported, never acted on: the
+    # tool cannot tell a retirement from a rename.
+    def orphans
+      pages.keys.reject { |key| File.file?(File.expand_path(key, dir)) }
+    end
+
+    def writable?
+      return File.writable?(path) if File.exist?(path)
+
+      File.directory?(dir) && File.writable?(dir)
+    end
+
+    # Read-modify-write under an exclusive lock, so parallel invocations across a
+    # corpus cannot drop each other's entries.
+    def save
+      @lock.synchronize do
+        raise Error, unwritable_message unless writable?
+
+        # Lock the manifest itself rather than a sidecar: a stray .lock file next to
+        # committed state is litter, and one more thing to gitignore.
+        File.open(path, File::RDWR | File::CREAT, 0o644) do |file|
+          file.flock(File::LOCK_EX)
+          merged = merge_with_disk
+          file.rewind
+          file.truncate(0)
+          file.write(HEADER + YAML.dump(sorted(merged)))
+          @data = merged
+        end
+        @created = false
+      end
+    end
+
+    private
+
+    def merge_with_disk
+      on_disk = File.file?(path) ? load_file : {}
+      merged = on_disk.merge(@data)
+      merged["pages"] = (on_disk["pages"] || {}).merge(@data["pages"] || {})
+      @forgotten.each { |key| merged["pages"].delete(key) }
+      merged.compact
+    end
+
+    # Sorted keys keep a one-document publish to a one-entry diff.
+    def sorted(data)
+      out = {}
+      out["workspace_id"] = data["workspace_id"] if data["workspace_id"]
+      out["pages"] = data["pages"].sort.to_h if data["pages"] && !data["pages"].empty?
+      out
+    end
+
+    def load_file
+      loaded = YAML.safe_load_file(path, permitted_classes: [], aliases: false)
+      raise ConfigError, malformed_message("expected a YAML mapping") unless loaded.nil? || loaded.is_a?(Hash)
+
+      loaded || {}
+    rescue Psych::Exception => e
+      raise ConfigError, malformed_message(e.message)
+    end
+
+    # Refuse rather than starting from an empty manifest, which would republish
+    # everything as new and duplicate the lot.
+    def malformed_message(detail)
+      <<~MSG.strip
+        #{path} could not be read: #{detail}
+
+        Publishing has stopped.
+        Starting from an empty manifest would duplicate every published document.
+        Fix the file, or delete it and re-adopt the existing pages deliberately.
+      MSG
+    end
+
+    def unwritable_message
+      <<~MSG.strip
+        Cannot write #{path}.
+
+        Checked before publishing on purpose: creating pages and then failing to
+        record them is the one failure that cannot be retried safely.
+      MSG
+    end
+  end
+end

data/lib/notion_publish/media.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module NotionPublish
+  # Finds local image references in a Markdown body.
+  #
+  # Notion's `markdown` parameter can only express *external* file references.
+  # A local path, an upload id, or anything else produces an image block with an
+  # empty URL -- and it does so without an error, so a dropped diagram looks
+  # like a successful publish. Local images therefore have to be uploaded and
+  # attached through the block API instead.
+  #
+  # To keep the server-side Markdown parser for everything else, each local
+  # image is swapped for a sentinel paragraph before publishing. The sentinel is
+  # located afterwards and the real image block is inserted in its place.
+  class Media
+    FENCE = /\A\s*(?:`{3,}|~{3,})/
+    # An image alone on its line. Only these can become blocks: Notion has no
+    # inline image, so an image sitting inside a sentence has nowhere to go.
+    STANDALONE = /\A\s*!\[([^\]]*)\]\(\s*([^)\s]+?)\s*(?:"[^"]*")?\s*\)\s*\z/
+    ANY_IMAGE = /!\[[^\]]*\]\([^)]*\)/
+    EXTERNAL = %r{\Ahttps?://}i
+
+    Image = Data.define(:index, :alt, :path) do
+      def sentinel = format("NOTIONPUBLISHIMAGE%04d", index)
+    end
+
+    attr_reader :images, :inline_paths
+
+    def self.scan(body, base_dir:) = new(body, base_dir: base_dir)
+
+    def initialize(body, base_dir:)
+      @body = body.to_s
+      @base_dir = base_dir
+      @images = []
+      @inline_paths = []
+      @rewritten = rewrite
+    end
+
+    def any? = !images.empty?
+
+    # The body with each local image replaced by its sentinel, ready to send as
+    # the `markdown` parameter.
+    def body_with_sentinels = @rewritten
+
+    def resolved_path(image) = File.expand_path(image.path, @base_dir)
+
+    private
+
+    def rewrite
+      in_fence = false
+      index = 0
+
+      lines = @body.lines.map do |line|
+        if line.match?(FENCE)
+          in_fence = !in_fence
+          next line
+        end
+        next line if in_fence
+
+        match = line.match(STANDALONE)
+        if match && local?(match[2])
+          image = Image.new(index: index, alt: match[1], path: unescape(match[2]))
+          @images << image
+          index += 1
+          next "#{line[/\A\s*/]}#{image.sentinel}\n"
+        end
+
+        # An image among other text cannot become a block. Note it so the caller
+        # can say so rather than letting it publish as an empty URL.
+        collect_inline(line) if match.nil? && line.match?(ANY_IMAGE)
+        line
+      end
+
+      lines.join
+    end
+
+    def collect_inline(line)
+      line.scan(/!\[[^\]]*\]\(\s*([^)\s]+?)\s*(?:"[^"]*")?\s*\)/) do |(url)|
+        @inline_paths << url if local?(url)
+      end
+    end
+
+    # Anything that is not a well-formed http(s) URL has to be treated as local.
+    # A malformed URL produces the same silent empty-URL block that a relative
+    # path does, so guessing charitably here would hide the failure.
+    def local?(url) = !url.to_s.match?(EXTERNAL)
+
+    def unescape(url) = url.to_s.gsub("%20", " ").delete_prefix("<").delete_suffix(">")
+  end
+end

data/lib/notion_publish/notion_digest.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module NotionPublish
+  # The hash of a page as Notion returns it, used to notice edits made in
+  # Notion.
+  #
+  # An uploaded file comes back as a signed S3 URL whose query string is
+  # minted fresh on every read, so the raw Markdown of any page with an
+  # uploaded image differs from one read to the next. The signature is
+  # removed before hashing; the path, which names the file, is kept. Pages
+  # without such URLs hash exactly as their raw Markdown does.
+  module NotionDigest
+    SIGNED_QUERY = %r{(https://[^\s)"'<>]+?)\?[^\s)"'<>]*X-Amz-[^\s)"'<>]*}
+
+    module_function
+
+    def of(markdown) = Digest::SHA256.hexdigest(stable(markdown))
+
+    def stable(markdown) = markdown.to_s.gsub(SIGNED_QUERY, '\1')
+  end
+end

data/lib/notion_publish/pool.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module NotionPublish
+  # Runs work over a list a few items at a time, and hands the results back
+  # in the list's order.
+  #
+  # Checking a page is almost all waiting on Notion, so a few requests in
+  # flight cut the time roughly by that factor. Notion allows about three
+  # requests a second on average; more workers than that would mostly buy
+  # rate-limit retries.
+  #
+  # Only the work runs on worker threads. The +started+ and +finished+
+  # callbacks and the block run on the calling thread, so everything that
+  # prints does so from one thread, in order.
+  class Pool
+    SIZE = 3
+
+    # Calls +work+ with each item on a worker thread. On the calling thread,
+    # calls +started+ with an item when a worker picks it up, +finished+ with
+    # the number of items done so far, and yields each item with its result
+    # in the original order as soon as every item before it is done. Returns
+    # the results in order.
+    #
+    # An exception raised by +work+ stops the run and is raised here, once
+    # the items before it have been yielded.
+    def self.run(items, work:, size: SIZE, started: nil, finished: nil, &emit)
+      new(items, work: work, started: started, finished: finished, emit: emit).run(size)
+    end
+
+    def initialize(items, work:, started:, finished:, emit:)
+      @items = items
+      @work = work
+      @started = started
+      @finished = finished
+      @emit = emit
+      @results = Array.new(items.length)
+      @events = Queue.new
+    end
+
+    def run(size)
+      return [] if @items.empty?
+
+      workers = Array.new([size, @items.length].min) { worker(jobs) }
+      collect
+      @results.map { |_, value| value }
+    ensure
+      workers&.each(&:kill)
+    end
+
+    private
+
+    def jobs
+      @jobs ||= Queue.new.tap do |queue|
+        @items.each_with_index { |item, index| queue << [item, index] }
+        queue.close
+      end
+    end
+
+    def worker(queue)
+      Thread.new do
+        while (job = queue.pop)
+          item, index = job
+          @events << [:started, index]
+          @results[index] = attempt(item)
+          @events << [:finished, index]
+        end
+      end
+    end
+
+    def attempt(item)
+      [:ok, @work.call(item)]
+    rescue StandardError => e
+      [:raised, e]
+    end
+
+    # Until every item has reported finishing, not merely until every result
+    # is in: a worker stores its result just before it says so.
+    def collect
+      done = 0
+      emitted = 0
+      while done < @items.length
+        kind, index = @events.pop
+        next @started&.call(@items[index]) if kind == :started
+
+        done += 1
+        @finished&.call(done)
+        emitted = emit_ready(emitted)
+      end
+    end
+
+    # Yields every finished result that has no unfinished item before it.
+    def emit_ready(emitted)
+      while emitted < @items.length && @results[emitted]
+        status, value = @results[emitted]
+        raise value if status == :raised
+
+        @emit&.call(@items[emitted], value)
+        emitted += 1
+      end
+      emitted
+    end
+  end
+end

data/lib/notion_publish/progress.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "delegate"
+require "io/console"
+
+module NotionPublish
+  # A single progress line on a terminal, such as "Checking 12/38  a.md":
+  # how many items are finished, and one that is in progress. It is rewritten
+  # in place and cleared when the run ends.
+  #
+  # It exists only when stderr is a terminal, following git, curl, and rsync,
+  # so CI logs and pipes never see it. Output written through #wrap clears the
+  # line first and redraws it afterwards, so results and the progress line
+  # never share a line. A disabled Progress does nothing and wraps nothing.
+  class Progress
+    CLEAR = "\r\e[K"
+    DEFAULT_WIDTH = 80
+
+    # An IO that keeps the progress line out of the way of what it writes.
+    class Output < SimpleDelegator
+      def initialize(io, progress)
+        super(io)
+        @progress = progress
+      end
+
+      def puts(*) = @progress.around { __getobj__.puts(*) }
+      def print(*) = @progress.around { __getobj__.print(*) }
+      def write(*) = @progress.around { __getobj__.write(*) }
+    end
+
+    def initialize(io, enabled:)
+      @io = io
+      @enabled = enabled
+      @line = nil
+    end
+
+    def enabled? = @enabled
+
+    def wrap(io) = enabled? ? Output.new(io, self) : io
+
+    def start(verb, total)
+      @verb = verb
+      @total = total
+      @done = 0
+      @label = ""
+    end
+
+    # Names an item being worked on. With several in flight, the line shows
+    # the one picked up most recently.
+    def started(label)
+      @label = label
+      redraw
+    end
+
+    # Counts finished items, so the number only rises and reaches the total
+    # exactly when the work is done.
+    def finished(done)
+      @done = done
+      redraw
+    end
+
+    # Clears the line for good.
+    def finish
+      return unless enabled? && @line
+
+      @io.write(CLEAR)
+      @line = nil
+    end
+
+    def around
+      return yield unless enabled? && @line
+
+      @io.write(CLEAR)
+      result = yield
+      draw
+      result
+    end
+
+    private
+
+    def redraw
+      return unless enabled?
+
+      @line = fit("#{@verb} #{@done}/#{@total}  #{@label}")
+      draw
+    end
+
+    def draw
+      @io.write("#{CLEAR}#{@line}")
+      @io.flush
+    end
+
+    # A line wider than the terminal wraps, and a wrapped line cannot be
+    # rewritten in place.
+    def fit(text)
+      width = (@io.winsize[1] if @io.respond_to?(:winsize)).to_i
+      width = DEFAULT_WIDTH unless width > 10
+      text.length < width ? text : "#{text[0, width - 4]}..."
+    rescue SystemCallError, IOError
+      text[0, DEFAULT_WIDTH - 1]
+    end
+  end
+end

data/lib/notion_publish/property_set.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require "date"
+require "json"
+require "time"
+
+require_relative "errors"
+
+module NotionPublish
+  # The requested property values, before the schema has been consulted.
+  #
+  # Every value is held as an array of strings. Repeating --property with the
+  # same name accumulates; whether that is legal depends on the property's type,
+  # which Schema decides later.
+  class PropertySet
+    include Enumerable
+
+    FRONT_MATTER = "front matter"
+
+    def self.parse_pair(pair)
+      name, separator, value = pair.to_s.partition("=")
+      raise Error, pair_message(pair) if separator.empty? || name.strip.empty?
+
+      [name.strip, value]
+    end
+
+    def self.pair_message(pair)
+      <<~MSG.strip
+        Cannot read #{pair.to_s.inspect} as a property.
+
+        Use --property 'Name=Value'. Repeat it to give a multi-select or people
+        property more than one value:
+
+          --property 'Function=Operations' --property 'Function=Legal'
+      MSG
+    end
+
+    # Layered lowest to highest: front matter, then --properties-json, then
+    # --property. Each layer replaces a name outright rather than adding to it,
+    # so overriding one property never drags the old values along.
+    def self.build(front_matter: nil, json: nil, pairs: [])
+      set = new
+      set.merge_hash(front_matter, source: FRONT_MATTER)
+      set.merge_hash(parse_json(json), source: "--properties-json") if json
+      set.merge_pairs(pairs)
+      set
+    end
+
+    def self.parse_json(json)
+      parsed = JSON.parse(json.to_s)
+      raise Error, "--properties-json must be a JSON object, got #{parsed.class}." unless parsed.is_a?(Hash)
+
+      parsed
+    rescue JSON::ParserError => e
+      raise Error, "Could not parse --properties-json: #{e.message}"
+    end
+
+    def initialize
+      @values = {}
+      @explicit = []
+    end
+
+    def merge_hash(hash, source:)
+      return self if hash.nil? || hash.empty?
+      raise Error, "#{source} properties must be a mapping." unless hash.is_a?(Hash)
+
+      hash.each do |name, value|
+        @values[name.to_s] = normalise(value)
+        @explicit << name.to_s if source != FRONT_MATTER
+      end
+      self
+    end
+
+    # Properties named on the command line were asked for by this invocation;
+    # properties from front matter belong to the document. That matters when a
+    # destination cannot hold them: an explicit one is an error, a document one
+    # is skipped with a warning.
+    def explicit?(name) = @explicit.include?(name.to_s)
+
+    def merge_pairs(pairs)
+      seen = {}
+      Array(pairs).each do |pair|
+        name, value = self.class.parse_pair(pair)
+        # First --property for a name replaces any lower layer; later ones add.
+        @values[name] = seen[name] ? @values[name] + [value] : [value]
+        seen[name] = true
+        @explicit << name
+      end
+      self
+    end
+
+    def each(&) = @values.each(&)
+    def empty? = @values.empty?
+    def names = @values.keys
+    def [](name) = @values[name]
+
+    private
+
+    def normalise(value)
+      case value
+      when nil then [""]
+      when Array then value.map { |v| stringify(v) }
+      else [stringify(value)]
+      end
+    end
+
+    def stringify(value)
+      case value
+      when true then "true"
+      when false then "false"
+      when Date, Time then value.iso8601
+      else value.to_s
+      end
+    end
+  end
+end