notion_publish 0.1.0

data/lib/notion_publish/commands/command.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require_relative "../errors"
+require_relative "../manifest"
+require_relative "../pool"
+require_relative "../progress"
+require_relative "../property_set"
+require_relative "../reference"
+require_relative "../resolver"
+require_relative "../settings"
+require_relative "../target"
+
+module NotionPublish
+  module Commands
+    # Shared plumbing: the invocation's options and streams, and the lookups
+    # that more than one command needs.
+    class Command
+      PARENT_TYPES = %w[page_id database_id data_source_id].freeze
+      MAX_DEPTH = 10
+
+      def initialize(context)
+        @context = context
+      end
+
+      private
+
+      def options = @context.options
+      def stdout = @stdout ||= progress.wrap(@context.stdout)
+      def stderr = @stderr ||= progress.wrap(@context.stderr)
+      def stdin = @context.stdin
+      def client = @context.client
+      def progress = @progress ||= Progress.new(@context.stderr, enabled: progress_wanted?)
+      def jobs = options[:jobs] || Pool::SIZE
+
+      # Only on a terminal, and not when the request log is already showing
+      # activity or --json output is being read by a program.
+      def progress_wanted?
+        terminal = @context.stderr.respond_to?(:tty?) && @context.stderr.tty?
+        terminal && !options[:no_progress] && !logging_requests? && !options[:json]
+      end
+
+      # -v: list every file, including those where nothing happened.
+      def listing_everything? = options[:verbose].to_i >= 1
+      def logging_requests? = options[:verbose].to_i >= 2
+
+      def settings_for(path)
+        Settings.for(File.dirname(File.expand_path(path)))
+      end
+
+      # First match wins: command line, then front matter, then settings.
+      def resolve_destination(document, settings)
+        resolver = Resolver.new(client)
+
+        if (parent = options[:parent] || document.parent || settings.parent)
+          return resolver.resolve(parent)
+        end
+
+        name = options[:database] || document.database || settings.database
+        return resolver.resolve_database_name(name) if name
+
+        raise Error, no_destination_message
+      end
+
+      # The recorded parent is enough: the page is updated in place, and the
+      # parent only supplies the schema. An adopted entry records none, so ask
+      # Notion where the page lives.
+      def target_for(entry)
+        parent = entry.parent
+        return resolve_parent_of(entry) unless parent
+
+        Target.new(kind: parent["type"] == "page_id" ? :page : :data_source, id: parent["id"],
+                   title: parent["name"], database_id: nil, inline: nil)
+      end
+
+      # A page's parent can be a block inside another page, such as a toggle
+      # heading the page was moved under. Walk up until a page, database, or
+      # data source appears; that is what supplies the schema.
+      def resolve_parent_of(entry)
+        parent = client.get("/v1/pages/#{entry.id}")["parent"] || {}
+        MAX_DEPTH.times do
+          type = parent["type"]
+          return Resolver.new(client).resolve_reference(parent[type]) if PARENT_TYPES.include?(type)
+          break unless type == "block_id"
+
+          parent = client.get("/v1/blocks/#{parent['block_id']}")["parent"] || {}
+        end
+        raise Error, "Cannot tell which page or database #{entry.url} sits under."
+      end
+
+      def property_set(document)
+        PropertySet.build(front_matter: document.properties, json: options[:properties_json],
+                          pairs: options[:properties] || [])
+      end
+
+      def manifest_in(dir)
+        Manifest.locate_in(File.expand_path(dir), override: options[:manifest])
+      end
+
+      def no_destination_message
+        <<~MSG.strip
+          No destination. Notion needs a parent for every page, and an internal
+          connection cannot create pages at the workspace root.
+
+          Give one of these:
+            --parent <ID, Notion URL, or exact database name>
+            --database <exact database name>
+
+          Or set notion_parent / notion_database in the file's front matter, or
+          parent / database in #{Settings::FILENAME}.
+        MSG
+      end
+    end
+  end
+end

data/lib/notion_publish/commands/properties.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "command"
+require_relative "../document"
+require_relative "../schema"
+
+module NotionPublish
+  module Commands
+    # Lists the destination's properties, their types, and the options a
+    # select can take, so a user can see what --property will accept.
+    class Properties < Command
+      def call
+        target = resolve_destination(Document.new("(none)", ""), Settings.for(Dir.pwd))
+        schema = Schema.for(client, target)
+
+        stdout.puts target.describe
+        schema.properties.each { |name, definition| print_property(name, definition) }
+        CLI::OK
+      end
+
+      private
+
+      def print_property(name, definition)
+        type = definition["type"]
+        note = Schema::COMPUTED.include?(type) ? " (read-only)" : ""
+        stdout.puts format("  %-24s %s%s", name, type, note)
+
+        choices = definition.dig(type, "options")
+        stdout.puts "    #{choices.map { |o| o['name'] }.join(' | ')}" if choices && !choices.empty?
+      end
+    end
+  end
+end

data/lib/notion_publish/commands/publish.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require "digest"
+
+require_relative "command"
+require_relative "reporting"
+require_relative "../decoration"
+require_relative "../document"
+require_relative "../publisher"
+
+module NotionPublish
+  module Commands
+    # Publishes one Markdown file, or with --dry-run says what publishing it
+    # would do.
+    class Publish < Command
+      include Reporting
+
+      # How a dry run shows each kind of property value.
+      SUMMARIES = {
+        "title" => ->(value) { value.map { |v| v.dig("text", "content") }.join.inspect },
+        "rich_text" => ->(value) { value.map { |v| v.dig("text", "content") }.join.inspect },
+        "multi_select" => ->(value) { value.map { |v| v["name"] }.inspect },
+        "select" => ->(value) { value["name"].inspect },
+        "status" => ->(value) { value["name"].inspect },
+        "people" => ->(value) { value.map { |v| v["id"] }.inspect },
+        "relation" => ->(value) { value.map { |v| v["id"] }.inspect },
+        # The form --property takes, rather than a Hash#inspect, whose format
+        # differs between Ruby versions.
+        "date" => ->(value) { value && [value["start"], value["end"]].compact.join("..") }
+      }.freeze
+
+      def call(path)
+        document = Document.load(path)
+        settings = settings_for(path)
+        map = manifest_for(path)
+        target = destination(document, settings, map&.entry(File.expand_path(path)))
+        icon = options[:icon] || document.icon || settings.icon
+        cover = options[:cover] || document.cover || settings.cover
+        Decoration.check!(icon, :icon) if icon
+        Decoration.check!(cover, :cover) if cover
+
+        return dry_run(path, document, target, map, icon: icon, cover: cover) if options[:dry_run]
+
+        warnings = []
+        outcome = publisher.publish(document, target: target, manifest: map, properties: property_set(document),
+                                              warnings: warnings, icon: icon, cover: cover, **publish_flags)
+        report(path, outcome, target, warnings)
+      end
+
+      private
+
+      def publisher = @publisher ||= Publisher.new(client)
+
+      # A file the manifest already tracks is updated where its page is, so it needs
+      # no destination. One that is given anyway is still used.
+      def destination(document, settings, entry)
+        named = options[:parent] || options[:database] || document.parent || document.database ||
+                settings.parent || settings.database
+        return target_for(entry) if entry && !named
+
+        resolve_destination(document, settings)
+      end
+
+      def publish_flags
+        {
+          title: options[:title], title_given: !options[:title].nil?,
+          upload: options[:upload] != false, keep_h1: options[:keep_h1] == true,
+          force: options[:force] == true, force_properties: options[:force_properties] == true
+        }
+      end
+
+      # Without --link, a manifest is used only if one already exists; publishing
+      # never starts recording a corpus unasked.
+      def manifest_for(path)
+        return nil if options[:link] == false
+
+        map = Manifest.locate(path, override: options[:manifest])
+        return map unless map.created?
+        return nil unless options[:link] || options[:manifest]
+
+        stderr.puts "Creating #{map.path}"
+        map
+      end
+
+      # A dry run resolves, validates every property against the live schema,
+      # and says what a real run would do.
+      def dry_run(path, document, target, map, icon:, cover:)
+        warnings = []
+        built = preview_properties(document, target, warnings)
+        media = publisher.scan_media(document)
+        check_images(media)
+
+        warnings.each { |w| stderr.puts w }
+        stdout.puts "Would #{intended_action(path, map)} #{path} to #{target.describe}"
+        stdout.puts "  title: #{(options[:title] || document.title).inspect}"
+        built.each { |key, payload| stdout.puts "  #{key}: #{summarise(payload)}" }
+        media.images.each { |image| stdout.puts "  upload: #{image.path}" }
+        stdout.puts "  icon: #{icon}" if icon
+        stdout.puts "  cover: #{cover}" if cover
+        CLI::OK
+      end
+
+      def preview_properties(document, target, warnings)
+        set = property_set(document)
+        return {} if set.empty?
+
+        publisher.build_properties(publisher.schema_for(target), set, options[:title] || document.title,
+                                   !options[:title].nil?, warnings)
+      end
+
+      def check_images(media)
+        media.images.each do |image|
+          resolved = media.resolved_path(image)
+          next if File.file?(resolved)
+
+          raise Error, "No such image: #{image.path} (looked in #{File.dirname(resolved)})"
+        end
+      end
+
+      def intended_action(path, map)
+        entry = map&.entry(File.expand_path(path))
+        return "publish" unless entry
+
+        same = entry.source_sha256 == Digest::SHA256.hexdigest(File.binread(File.expand_path(path)))
+        same && options[:force] != true ? "leave unchanged" : "update"
+      end
+
+      def summarise(payload)
+        type, value = payload.first
+        SUMMARIES.fetch(type, :inspect.to_proc).call(value)
+      end
+    end
+  end
+end

data/lib/notion_publish/commands/relink.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require "digest"
+
+require_relative "../notion_digest"
+require_relative "command"
+require_relative "../links"
+require_relative "../pool"
+
+module NotionPublish
+  module Commands
+    # The second pass after publishing a set. A document published before the
+    # one it links to could not resolve that link, so Notion stored it as
+    # https://<filename>.md. Now that every page has a URL, fix those.
+    class Relink < Command
+      def call(dir)
+        map = manifest_in(dir)
+        if map.created? || map.pages.empty?
+          stderr.puts "Nothing published yet: no #{Manifest::FILENAME} with pages in #{dir}."
+          return CLI::FAILURE
+        end
+
+        fixed = relink_all(map).sum
+        report_orphans(map)
+        stdout.puts fixed.zero? ? "No links needed fixing." : "Fixed #{fixed} #{plural(fixed, 'link')}."
+        CLI::OK
+      end
+
+      private
+
+      # Several pages at a time. Workers read the published URLs from a copy
+      # taken first, since the manifest itself changes as pages are rehashed.
+      def relink_all(map)
+        urls = map.pages.transform_values { |raw| raw["url"] }
+        client
+        progress.start("Checking", map.pages.length)
+        Pool.run(map.pages.to_a, size: jobs, work: lambda { |(key, raw)|
+          relink_page(map, urls, key, Manifest::Entry.from(raw))
+        },
+                                 started: ->((key, _)) { progress.started(key) },
+                                 finished: ->(done) { progress.finished(done) }) do |(key, _), count|
+          stdout.puts "#{File.basename(key)}: fixed #{count} #{plural(count, 'link')}" if count.positive?
+        end
+      ensure
+        progress.finish
+      end
+
+      # Returns the number of links fixed.
+      def relink_page(map, urls, key, entry)
+        before = read_markdown(entry.id)
+        updates = pending_updates(urls, before)
+        return 0 if updates.empty?
+
+        client.patch("/v1/pages/#{entry.id}/markdown",
+                     { "type" => "update_content", "update_content" => { "content_updates" => updates } })
+        rehash(map, key, entry, before)
+        updates.length
+      end
+
+      # The page now differs from what was recorded after publishing, and it
+      # was this tool that changed it. Record the new content, or the next
+      # status would call it an edit made in Notion. Only when the page was in
+      # sync beforehand: an edit someone else made must stay visible.
+      def rehash(map, key, entry, before)
+        return unless entry.notion_sha256 == NotionDigest.of(before)
+
+        after = NotionDigest.of(read_markdown(entry.id))
+        map.record(File.expand_path(key, map.dir), entry.with(notion_sha256: after))
+      end
+
+      def read_markdown(page_id) = client.get("/v1/pages/#{page_id}/markdown")["markdown"].to_s
+
+      def pending_updates(urls, markdown)
+        urls.filter_map do |key, url|
+          mangled = Links.mangled(File.basename(key))
+          next unless markdown.include?("](#{mangled})")
+
+          { "old_str" => "](#{mangled})", "new_str" => "](#{url})", "replace_all_matches" => true }
+        end
+      end
+
+      # The tool cannot tell a retirement from a rename, so it reports and stops.
+      def report_orphans(map)
+        orphans = map.orphans
+        return if orphans.empty?
+
+        stderr.puts "#{orphans.length} #{orphans.length == 1 ? 'entry has' : 'entries have'} no source file:"
+        orphans.each { |key| stderr.puts "  #{key}" }
+        stderr.puts "Their Notion pages are still live. Remove them there if they are retired."
+      end
+
+      def plural(count, word) = count == 1 ? word : "#{word}s"
+    end
+  end
+end

data/lib/notion_publish/commands/reporting.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "json"
+
+module NotionPublish
+  module Commands
+    # How a publish outcome is shown, shared by publish and republish so a
+    # script sees the same lines and the same JSON from either.
+    module Reporting
+      private
+
+      # A single file you named is always reported. In a run over many files,
+      # the ones where nothing happened are listed only with -v.
+      def show_unchanged? = true
+
+      # Prints the outcome and returns the exit code it deserves on its own.
+      def report(path, outcome, target, warnings)
+        warnings.each { |w| stderr.puts w }
+        report_json(path, outcome, target) if options[:json]
+
+        case outcome.action
+        when :blocked
+          stderr.puts "Blocked #{path}"
+          stderr.puts outcome.detail
+          CLI::BLOCKED
+        when :skipped
+          stderr.puts "Skipped #{path}"
+          stderr.puts outcome.detail
+          CLI::BLOCKED
+        else
+          report_text(path, outcome, target) unless options[:json]
+          CLI::OK
+        end
+      end
+
+      def report_json(path, outcome, target)
+        stdout.puts JSON.generate(
+          "source" => path, "action" => outcome.action.to_s,
+          "id" => outcome.id, "url" => outcome.url,
+          "parent" => target.id, "parent_name" => target.title
+        )
+      end
+
+      def report_text(path, outcome, target)
+        case outcome.action
+        when :unchanged
+          stdout.puts "Unchanged #{path}" if show_unchanged?
+          return
+        when :properties then stdout.puts "Updated properties on #{path}"
+        when :created then stdout.puts "Published #{path} to #{target.describe}"
+        else stdout.puts "Updated #{path} to #{target.describe}"
+        end
+        stdout.puts "  #{outcome.url}"
+      end
+    end
+  end
+end

data/lib/notion_publish/commands/republish.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "command"
+require_relative "reporting"
+require_relative "../decoration"
+require_relative "../document"
+require_relative "../pool"
+require_relative "../publisher"
+require_relative "../target"
+
+module NotionPublish
+  module Commands
+    # Publishes every document the manifest tracks, from what each entry
+    # recorded, without naming files one by one.
+    #
+    # Each page is updated where it already is, so no destination is needed
+    # and none is accepted. Files the manifest does not track are left alone:
+    # publishing a new document stays something you do on purpose.
+    class Republish < Command
+      include Reporting
+
+      # Options that describe one document or one destination. They make no
+      # sense applied to a whole set, so they are reported and ignored.
+      IGNORED = {
+        parent: "--parent", database: "--database", properties: "--property",
+        properties_json: "--properties-json", title: "--title", icon: "--icon", cover: "--cover",
+        keep_h1: "--keep-h1", link: "--link/--no-link", page: "--page", yes: "--yes", local: "--local",
+        untracked: "--untracked"
+      }.freeze
+
+      SUMMARY = {
+        unchanged: "unchanged", updated: "updated", properties: "properties updated", created: "recreated",
+        blocked: "changed in Notion", skipped: "skipped", failed: "failed", orphaned: "no source file"
+      }.freeze
+
+      def call(dir)
+        return dry_run_refused if options[:dry_run]
+
+        warn_ignored
+        map = manifest_in(dir)
+        if map.created? || map.pages.empty?
+          stderr.puts "Nothing published yet: no #{Manifest::FILENAME} with pages in #{dir}."
+          return CLI::FAILURE
+        end
+
+        entries = entries_under(map, File.expand_path(dir)).to_a
+        results = run_all(map, entries).map(&:action)
+        stdout.puts summary(results) unless options[:json]
+        exit_code(results)
+      end
+
+      private
+
+      def publisher = @publisher ||= Publisher.new(client)
+      def show_unchanged? = listing_everything?
+
+      def dry_run_refused
+        stderr.puts "republish has no --dry-run. `notion-publish status` shows what it would do."
+        CLI::USAGE
+      end
+
+      def warn_ignored
+        given = IGNORED.filter_map { |key, flag| flag if options.key?(key) }
+        return if given.empty?
+
+        stderr.puts "Ignoring #{given.join(', ')}: republish updates each page where it already is, " \
+                    "from what its entry recorded."
+      end
+
+      def entries_under(map, root)
+        map.pages.select do |key, _|
+          path = File.expand_path(key, map.dir)
+          path == root || path.start_with?("#{root}#{File::SEPARATOR}")
+        end
+      end
+
+      # What republishing one file came to. Built on a worker thread and
+      # printed on the calling one, in file order.
+      Result = Data.define(:key, :action, :outcome, :target, :warnings, :message)
+
+      # Several files at a time. The client, the publisher, and the manifest are
+      # set up here first, so no worker races to create them.
+      def run_all(map, entries)
+        client
+        publisher
+        progress.start("Checking", entries.length)
+        Pool.run(entries, size: jobs, work: ->((key, raw)) { republish(map, key, raw) },
+                          started: ->((key, _)) { progress.started(key) },
+                          finished: ->(done) { progress.finished(done) }) do |_, result|
+          print_result(result)
+        end
+      ensure
+        progress.finish
+      end
+
+      # One document failing does not stop the rest, the same as a shell loop
+      # without `|| break`.
+      def republish(map, key, raw)
+        entry = Manifest::Entry.from(raw)
+        path = File.expand_path(key, map.dir)
+        return Result.new(key, :orphaned, nil, nil, [], nil) unless File.file?(path)
+
+        target = target_for(entry)
+        warnings = []
+        outcome = publish(path, entry, target, map, warnings)
+        Result.new(key, outcome.action, outcome, target, warnings, nil)
+      rescue Error => e
+        Result.new(key, :failed, nil, nil, [], e.message)
+      end
+
+      def print_result(result)
+        return report(result.key, result.outcome, result.target, result.warnings) if result.outcome
+
+        # A script reading --json must see every file, including those that
+        # never reached Notion.
+        if options[:json]
+          stdout.puts JSON.generate({ "source" => result.key, "action" => result.action.to_s,
+                                      "error" => result.message }.compact)
+        end
+        case result.action
+        when :orphaned then stderr.puts "Skipped #{result.key}: no source file. Its Notion page is still live."
+        when :failed then stderr.puts "Failed #{result.key}: #{result.message}"
+        end
+      end
+
+      def publish(path, entry, target, map, warnings)
+        document = Document.load(path)
+        settings = settings_for(path)
+        icon = document.icon || settings.icon
+        cover = document.cover || settings.cover
+        Decoration.check!(icon, :icon) if icon
+        Decoration.check!(cover, :cover) if cover
+
+        publisher.republish(document, entry: entry, target: target, manifest: map, warnings: warnings,
+                                      upload: options[:upload] != false, icon: icon, cover: cover,
+                                      force: options[:force] == true,
+                                      force_properties: options[:force_properties] == true)
+      end
+
+      def summary(results)
+        counts = results.tally
+        parts = SUMMARY.filter_map { |action, label| "#{counts[action]} #{label}" if counts[action] }
+        "#{results.length} #{results.length == 1 ? 'document' : 'documents'}: #{parts.join(', ')}"
+      end
+
+      def exit_code(results)
+        return CLI::FAILURE if results.include?(:failed)
+        return CLI::BLOCKED if results.intersect?(%i[blocked skipped])
+
+        CLI::OK
+      end
+    end
+  end
+end

data/lib/notion_publish/commands/status.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "command"
+require_relative "../status"
+
+module NotionPublish
+  module Commands
+    # What would happen if you published everything. Exits 3 when a tracked
+    # document needs something done to it, so it works as a CI check. Files
+    # that were never published do not count: plenty are deliberately not
+    # mirrored.
+    class Status < Command
+      # Problems first, then what is in sync, then what was never published.
+      ORDER = (NotionPublish::Status::ACTIONABLE + %i[unchanged unpublished]).freeze
+      # States where the page itself is what needs looking at.
+      SHOW_URL = %i[drifted diverged missing trashed orphaned].freeze
+
+      def call(dir)
+        map = manifest_in(dir)
+        if map.created?
+          stderr.puts "Nothing published yet: no #{Manifest::FILENAME} at or above #{dir}."
+          return CLI::FAILURE
+        end
+
+        check = options[:local] != true
+        progress.start("Checking", map.pages.length)
+        rows = NotionPublish::Status.new(map, client: check ? client : nil)
+                                    .rows(dir: dir, check_notion: check, jobs: jobs,
+                                          started: ->(key) { progress.started(key) },
+                                          finished: ->(done) { progress.finished(done) })
+        progress.finish
+        options[:json] ? print_json(rows) : print_text(map, rows, check)
+        rows.any?(&:actionable?) ? CLI::BLOCKED : CLI::OK
+      end
+
+      private
+
+      def print_json(rows)
+        rows.each do |row|
+          stdout.puts JSON.generate("source" => row.source, "state" => row.state.to_s, "url" => row.url)
+        end
+      end
+
+      def print_text(map, rows, checked)
+        tracked = rows.count { |r| r.state != :unpublished }
+        stdout.puts "#{map.path} -- #{tracked} tracked"
+        stdout.puts "(not checked against Notion; --local was given)" unless checked
+
+        groups = rows.group_by(&:state)
+        ORDER.each { |state| print_group(state, groups[state]) if groups[state] }
+
+        stdout.puts
+        stdout.puts summary(groups.transform_values(&:length))
+      end
+
+      def print_group(state, rows)
+        return if state == :unchanged && !listing_everything?
+
+        label = NotionPublish::Status::STATES[state]
+        heading = "#{label[0].upcase}#{label[1..]} (#{rows.length})"
+        stdout.puts
+        if state == :unpublished && !options[:untracked]
+          stdout.puts "#{heading}: pass --untracked to list them"
+          return
+        end
+
+        stdout.puts heading
+        rows.each do |row|
+          stdout.puts "  #{row.source}"
+          stdout.puts "    #{row.url}" if row.url && SHOW_URL.include?(state)
+        end
+      end
+
+      def summary(counts)
+        return "Everything is in sync." if counts.empty? || counts.keys == [:unchanged]
+
+        NotionPublish::Status::STATES.filter_map { |state, label| "#{counts[state]} #{label}" if counts[state] }
+                                     .join(", ")
+      end
+    end
+  end
+end