open_blog 0.1.0

data/lib/open_blog/renderer/scrubber.rb

@@ -0,0 +1,35 @@
+module OpenBlog
+  class Renderer
+    class Scrubber < Rails::HTML::PermitScrubber
+      def initialize(metadata)
+        super()
+        self.tags = Sanitizer::TAGS
+        self.attributes = Sanitizer::ATTRIBUTES + %w[type checked disabled] + [ Sanitizer::TOKEN ]
+        @metadata = metadata
+      end
+
+      def scrub(node)
+        if node.element? && ((node.name == "input" && (node["type"] != "checkbox" || !node.key?("disabled"))) ||
+            (node.name == "img" && !Sanitizer.safe_url?(node["src"], image: true)))
+          node.remove
+          return STOP
+        end
+        super
+      end
+
+      protected
+
+      def scrub_attribute(node, attribute)
+        name = attribute.name
+        invalid_input = %w[type checked disabled].include?(name) && node.name != "input"
+        invalid_token = name == Sanitizer::TOKEN && !@metadata.key?(attribute.value)
+        invalid_url = %w[href src].include?(name) && !Sanitizer.safe_url?(attribute.value, image: name == "src")
+        if invalid_input || invalid_token || invalid_url
+          attribute.remove
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/open_blog/renderer/transform.rb

@@ -0,0 +1,115 @@
+require "set"
+
+module OpenBlog
+  class Renderer
+    class Transform
+      def initialize(document, post: nil)
+        @fragment = Sanitizer.restore!(document)
+        @post = post
+      end
+
+      def call
+        headings = @fragment.css("h1,h2,h3,h4,h5,h6")
+        shift = headings.any? ? 2 - headings.map { |node| node.name[1].to_i }.min : 0
+        used_ids = @fragment.css("[id]").map { |node| node["id"] }.to_set
+        @fragment.css("h1,h2,h3,h4,h5,h6,pre,img,table,a").each do |node|
+          case node.name
+          when /\Ah[1-6]\z/ then heading(node, shift, used_ids)
+          when "pre" then Highlighter.call(node)
+          when "img" then image(node)
+          when "table" then table(node)
+          when "a" then link(node)
+          end
+        end
+        @fragment
+      end
+
+      private
+
+      def heading(node, shift, used_ids)
+        node.name = "h#{[ node.name[1].to_i + shift, 6 ].min}"
+        base = node.text.parameterize.presence || "section"
+        identifier, suffix = base, 0
+        while used_ids.include?(identifier)
+          suffix += 1
+          identifier = "#{base}-#{suffix}"
+        end
+        used_ids << identifier
+        node["id"] = identifier
+        anchor = Nokogiri::XML::Node.new("a", node.document)
+        anchor["class"] = "ob-heading-anchor"
+        anchor["href"] = "##{identifier}"
+        anchor["aria-label"] = I18n.t("open_blog.headings.link", heading: node.text, locale: OpenBlog.config.locale)
+        anchor.content = "#"
+        node.add_child(anchor)
+      end
+
+      def image(node)
+        stored = ImageResolution.image_for_url(node["src"], post: @post)
+        if stored
+          node["src"] = stored.path
+          node["width"] = stored.width.to_s if stored.width
+          node["height"] = stored.height.to_s if stored.height
+        end
+        node["alt"] ||= ""
+        node["loading"] = "lazy"
+        figure = node.ancestors("figure").first
+        unless figure
+          figure = Nokogiri::XML::Node.new("figure", node.document)
+          node.replace(figure)
+          figure.add_child(node)
+          lift_figure(figure)
+        end
+        figure["class"] = "ob-figure"
+        if node["title"].present? && !figure.at_css("figcaption")
+          caption = Nokogiri::XML::Node.new("figcaption", node.document)
+          caption.content = node["title"]
+          figure.add_child(caption)
+        end
+      end
+
+      def lift_figure(figure)
+        while figure.parent && %w[p a strong em del span sup sub].include?(figure.parent.name)
+          parent = figure.parent
+          before, after = copy_wrapper(parent), copy_wrapper(parent)
+          figure.xpath("preceding-sibling::node()").each { |sibling| before.add_child(sibling) }
+          figure.xpath("following-sibling::node()").each { |sibling| after.add_child(sibling) }
+          unless parent.name == "p"
+            wrapper = copy_wrapper(parent)
+            figure.children.each { |child| wrapper.add_child(child) }
+            figure.add_child(wrapper)
+          end
+          parent.add_previous_sibling(before) if before.children.any?
+          parent.add_previous_sibling(figure)
+          parent.add_previous_sibling(after) if after.children.any?
+          parent.remove
+        end
+      end
+
+      def copy_wrapper(node)
+        node.dup(0).tap do |copy|
+          node.attribute_nodes.each { |attribute| copy[attribute.name] = attribute.value }
+        end
+      end
+
+      def table(node)
+        wrapper = Nokogiri::XML::Node.new("div", node.document)
+        wrapper["class"] = "ob-table"
+        wrapper["tabindex"] = "0"
+        node.replace(wrapper)
+        wrapper.add_child(node)
+      end
+
+      def link(node)
+        return if node["href"].blank?
+        uri = URI.parse(node["href"])
+        return unless uri.host
+        base = URI.parse(OpenBlog.config.public_base_url) if OpenBlog.config.public_base_url
+        return if base && uri.host.casecmp?(base.host) && uri.port == base.port
+        node["rel"] = (node["rel"].to_s.split + [ "noopener" ]).uniq.join(" ")
+      rescue URI::InvalidURIError
+        nil
+      end
+    end
+  end
+end

data/lib/open_blog/renderer.rb

@@ -0,0 +1,51 @@
+require "digest"
+
+module OpenBlog
+  class Renderer
+    VERSION = 2
+    autoload :Markdown, "open_blog/renderer/markdown"
+    autoload :RichText, "open_blog/renderer/rich_text"
+    autoload :Sanitizer, "open_blog/renderer/sanitizer"
+    autoload :Scrubber, "open_blog/renderer/scrubber"
+    autoload :Transform, "open_blog/renderer/transform"
+    autoload :Highlighter, "open_blog/renderer/highlighter"
+
+    class << self
+      def render(post, cache: true)
+        return render_string(post.body_for_payload, format: post.body_format, post: post) unless post.persisted? && cache
+        options = [ OpenBlog.config.markdown_hardbreaks, OpenBlog.config.public_base_url, OpenBlog.config.locale.to_s ]
+        configuration = Digest::SHA256.hexdigest(JSON.generate(options))
+        Rails.cache.fetch([ "open_blog/body", post.id, post.current_revision_identifier, VERSION, configuration ]) do
+          render_string(post.body_for_payload, format: post.body_format, post: post)
+        end.html_safe
+      end
+
+      def render_string(text, format:, post: nil)
+        parsed = sanitized(text, format: format, post: post)
+        Transform.new(parsed, post: post).call.to_html.html_safe
+      end
+
+      def document(post)
+        parsed = sanitized(post.body_for_payload, format: post.body_format, post: post)
+        Sanitizer.restore!(parsed)
+      end
+
+      def body_images(post)
+        ImageResolution.body_images(post)
+      end
+
+      private
+
+      def sanitized(text, format:, post:)
+        case format.to_s
+        when "markdown"
+          Sanitizer.call(Markdown.render(text), trusted: true)
+        when "rich_text"
+          Sanitizer.call(RichText.render(text, post: post))
+        else
+          raise ArgumentError, "Unsupported body format: #{format}"
+        end
+      end
+    end
+  end
+end

data/lib/open_blog/result.rb

@@ -0,0 +1,11 @@
+module OpenBlog
+  Result = Data.define(:post, :created, :records, :label, :findings, :error, :dry_run) do
+    def initialize(post: nil, created: false, records: { revision: nil, publication: nil, approval: nil }, label: :ai_unknown, findings: [], error: nil, dry_run: false)
+      super
+    end
+
+    def success?
+      error.nil?
+    end
+  end
+end

data/lib/open_blog/revision_payload.rb

@@ -0,0 +1,67 @@
+require "digest"
+require "json"
+
+module OpenBlog
+  class RevisionPayload
+    VERSION = "1".freeze
+
+    def self.from_fields(title:, description:, search_title:, search_description:, body:, author:, images:, faq:)
+      new(nil, title: title, description: description, search_title: search_title,
+        search_description: search_description, body: body, author: author, images: images, faq: faq)
+    end
+
+    def initialize(post, **fields)
+      fields = fields_from_post(post) if post
+      @payload = {
+        "author" => normalize(fields[:author]),
+        "body" => normalize(fields[:body], body: true),
+        "description" => normalize(fields[:description]),
+        "faq" => fields.fetch(:faq).map { |entry| normalized_entry(entry, %w[answer question]) },
+        "images" => fields.fetch(:images).map { |entry| normalized_entry(entry, %w[alt role sha256 url]) },
+        "search_description" => normalize(fields[:search_description]),
+        "search_title" => normalize(fields[:search_title]),
+        "title" => normalize(fields[:title]),
+        "version" => VERSION
+      }
+    end
+
+    def to_h
+      @payload
+    end
+
+    def to_json(*)
+      JSON.generate(@payload, ascii_only: false, script_safe: false)
+    end
+
+    def identifier
+      Digest::SHA256.hexdigest(to_json)
+    end
+
+    private
+
+    def fields_from_post(post)
+      images = []
+      images << image_entry(post.cover_image, "cover", post.cover_alt) if post.cover_image
+      images.concat(Renderer.body_images(post))
+      images << image_entry(post.social_image, "social", "") if post.social_image
+      {
+        title: post.title, description: post.description, search_title: post.search_title,
+        search_description: post.search_description, body: post.body_for_payload,
+        author: post.author_name, images: images, faq: post.faq_list
+      }
+    end
+
+    def image_entry(image, role, alt)
+      { role: role, url: image.path, sha256: image.sha256, alt: alt }
+    end
+
+    def normalized_entry(entry, keys)
+      keys.to_h { |key| [ key, normalize(entry.key?(key) ? entry[key] : entry[key.to_sym]) ] }
+    end
+
+    def normalize(value, body: false)
+      text = value.to_s.encode(Encoding::UTF_8).unicode_normalize(:nfc).gsub(/\r\n?/, "\n")
+      body ? text.sub(/\n+\z/, "") : text.gsub(/\A[ \t\n]+|[ \t\n]+\z/, "")
+    end
+  end
+end

data/lib/open_blog/rich_text_guard.rb

@@ -0,0 +1,26 @@
+module OpenBlog
+  module RichTextGuard
+    extend ActiveSupport::Concern
+    include ContentGuard
+
+    private
+
+    def content_guard_target?(attributes)
+      candidate = attributes.stringify_keys
+      candidate.fetch("record_type", record_type) == "OpenBlog::Post" && candidate.fetch("name", name) == "rich_body"
+    end
+
+    def content_guard_applies?
+      (record_type == "OpenBlog::Post" && name == "rich_body") ||
+        (attribute_in_database("record_type") == "OpenBlog::Post" && attribute_in_database("name") == "rich_body")
+    end
+
+    def content_guard_identity_changed?
+      persisted? && %w[record_type record_id name].any? { |attribute| will_save_change_to_attribute?(attribute) }
+    end
+
+    def content_guard_parent
+      record
+    end
+  end
+end

data/lib/open_blog/sample.rb

@@ -0,0 +1,56 @@
+require "digest"
+require "stringio"
+
+module OpenBlog
+  class Sample
+    EXTERNAL_ID = "open_blog_sample".freeze
+    ASSETS = File.expand_path("../generators/open_blog/install/templates", __dir__).freeze
+
+    def self.call
+      new.call
+    end
+
+    def call
+      existing = Post.find_by(external_id: EXTERNAL_ID)
+      return existing if existing
+
+      Post.transaction(requires_new: true) do
+        image = cover
+        markdown = File.read(File.join(ASSETS, "sample_post.md")).gsub("{{sample_image}}", image.path)
+        format = OpenBlog.config.default_body_format
+        body = format.to_s == "rich_text" ? rich_text(markdown) : markdown
+        result = Publish.call({ external_id: EXTERNAL_ID, title: "A small guide to your new blog",
+          slug: "open-blog-sample", description: "Explore headings, images, code and reader questions in this sample article.",
+          body: body, body_format: format, category: "General", provenance: "ai_assisted",
+          cover_image: image, cover_alt: "Blue and green blocks arranged as a simple landscape",
+          faq: [ { question: "Can I edit this sample?", answer: "Yes. Replace it with your own article, or remove it when you are ready." },
+            { question: "Will running the sample task create another copy?", answer: "No. Running the task again keeps the existing sample and its content." } ] },
+          post: Post.new, actor: "OpenBlog sample")
+        raise result.error unless result.success?
+        result.post
+      end.tap { @completed = true }
+    ensure
+      @uploaded_blob.service.delete(@uploaded_blob.key) if @uploaded_blob && !@completed
+    end
+
+    private
+
+    def rich_text(markdown)
+      document = Nokogiri::HTML5.fragment(Renderer::Markdown.render(markdown))
+      # Rich text removes source IDs. Keep footnote text without broken links.
+      document.css("a[data-footnote-ref]").each { |link| link.replace(link.children) }
+      document.css("a[data-footnote-backref]").remove
+      document.to_html
+    end
+
+    def cover
+      bytes = File.binread(File.join(ASSETS, "sample_cover.png"))
+      existing = Image.find_by(sha256: Digest::SHA256.hexdigest(bytes))
+      return existing if existing
+      @uploaded_blob = ActiveStorage::Blob.create_and_upload!(io: StringIO.new(bytes), filename: "sample-cover.png",
+        content_type: "image/png", identify: false,
+        service_name: OpenBlog.config.storage_service || Rails.application.config.active_storage.service)
+      ImageImport.prepare(signed_id: @uploaded_blob.signed_id).persist(uploaded_by: "OpenBlog sample")
+    end
+  end
+end

data/lib/open_blog/save_draft.rb

@@ -0,0 +1,7 @@
+module OpenBlog
+  class SaveDraft
+    def self.call(attributes, post: nil, actor:, now: Time.current, authorize: nil)
+      WritePost.call(attributes, post: post, actor: actor, now: now, publish: false, authorize: authorize)
+    end
+  end
+end

data/lib/open_blog/search.rb

@@ -0,0 +1,21 @@
+module OpenBlog
+  class Search
+    def self.call(query, scope: Post.listed, limit: nil)
+      return scope.none unless query.is_a?(String)
+      query = query.strip
+      return scope.none unless (2..100).cover?(query.length)
+
+      relation = if Post.connection.adapter_name == "PostgreSQL"
+        vector = "to_tsvector('simple', open_blog_posts.search_text)"
+        expression = Post.sanitize_sql_array([ "websearch_to_tsquery('simple', ?)", query ])
+        scope.where("#{vector} @@ #{expression}").reorder(Arel.sql("ts_rank(#{vector}, #{expression}) DESC"))
+      else
+        query.split.reduce(scope) do |matches, word|
+          matches.where("LOWER(open_blog_posts.search_text) LIKE ? ESCAPE '\\'", "%#{Post.sanitize_sql_like(word.downcase)}%")
+        end.reorder(nil)
+      end
+      relation = relation.order(created_at: :desc, id: :desc)
+      limit ? relation.limit(limit) : relation
+    end
+  end
+end

data/lib/open_blog/sitemap_entries.rb

@@ -0,0 +1,66 @@
+require "uri"
+
+module OpenBlog
+  class SitemapEntries
+    def self.call(base_url: OpenBlog.config.public_base_url)
+      raise ConfigurationError, "Set public_base_url or pass base_url to sitemap_entries" if base_url.blank?
+      new(base_url).call
+    end
+
+    def initialize(base_url)
+      unless base_url.is_a?(String)
+        raise ConfigurationError, "Sitemap base_url must be an absolute HTTP(S) origin"
+      end
+      @origin = URI.parse(base_url)
+      unless @origin.is_a?(URI::HTTP) && @origin.host.present? && @origin.userinfo.nil? &&
+          [ "", "/" ].include?(@origin.path) && @origin.query.nil? && @origin.fragment.nil?
+        raise ConfigurationError, "Sitemap base_url must be an absolute HTTP(S) origin"
+      end
+      @base_url = base_url.chomp("/")
+    rescue URI::InvalidURIError
+      raise ConfigurationError, "Sitemap base_url must be an absolute HTTP(S) origin"
+    end
+
+    def call
+      featured = ReaderQueries.featured
+      count = Post.listed.count - (featured ? 1 : 0)
+      entries = list_pages(OpenBlog.mount_path, count)
+      Post.listed.find_each do |post|
+        next if off_site?(post.canonical_url)
+        entry = { loc: post.url(base: @base_url) }
+        date = ReaderDates.modified_at(post)
+        entry[:lastmod] = date if date
+        entries << entry
+      end
+      Page.published.find_each do |page|
+        next if OpenBlog.config.policy_urls[page.kind.to_sym].present?
+        entries << { loc: page.url(base: @base_url), lastmod: page.updated_at }
+      end
+      if OpenBlog.config.primary_list_type == :tags
+        Tag.find_each do |tag|
+          entries.concat(list_pages(ReaderQueries.tag_path(tag), tag.posts.listed.count))
+        end
+      else
+        Category.find_each do |category|
+          entries.concat(list_pages(ReaderQueries.category_path(category), category.posts.listed.count))
+        end
+      end
+      entries
+    end
+
+    private
+
+    def list_pages(path, count)
+      total = [ (count.to_f / OpenBlog.config.posts_per_page).ceil, 1 ].max
+      (1..total).map { |page| { loc: "#{@base_url}#{path}#{page > 1 ? "?page=#{page}" : ''}" } }
+    end
+
+    def off_site?(url)
+      return false if url.blank?
+      uri = URI.parse(url)
+      uri.host&.downcase != @origin.host.downcase || uri.port != @origin.port
+    rescue URI::InvalidURIError
+      true
+    end
+  end
+end

data/lib/open_blog/surface_report/expected_content.rb

@@ -0,0 +1,27 @@
+module OpenBlog
+  class SurfaceReport
+    class ExpectedContent
+      class RenderingController < OpenBlog::ApplicationController
+        helper do
+          def open_blog_notices(*) = "".html_safe
+        end
+      end
+
+      def self.call(post, payload, base_url:)
+        snapshot = post.dup
+        %w[title description search_title search_description].each { |field| snapshot.public_send("#{field}=", payload.fetch(field)) }
+        snapshot.author = post.author
+        snapshot.author_name = payload.fetch("author")
+        snapshot.define_singleton_method(:body_for_payload) { payload.fetch("body") }
+        snapshot.define_singleton_method(:faq_list) { payload.fetch("faq").map(&:symbolize_keys) }
+        cover = payload.fetch("images").find { |entry| entry["role"] == "cover" }
+        snapshot.cover_image = cover && Image.find_by(sha256: cover["sha256"])
+        snapshot.cover_alt = cover&.fetch("alt")
+        page = ReaderPage.new(kind: :preview, record: snapshot, path: post.path, base_url: base_url)
+        html = RenderingController.render(template: "open_blog/posts/show", layout: false,
+          assigns: { open_blog_page: page, related_posts: [], related_cache_key: [] })
+        Nokogiri::HTML5(html).at_css("[data-open-blog-content]")
+      end
+    end
+  end
+end

data/lib/open_blog/surface_report/http.rb

@@ -0,0 +1,92 @@
+require "net/http"
+require "resolv"
+require "ipaddr"
+require "timeout"
+require_relative "../image_fetch"
+
+module OpenBlog
+  class SurfaceReport
+    class Http
+      Response = Struct.new(:status, :headers, :body, :url, keyword_init: true)
+      class FetchError < StandardError
+        def initialize(_message = nil) = super("Request unavailable")
+      end
+
+      def initialize(trusted_origin: nil, max_bytes: 10 * 1024 * 1024, timeout: 10, client: nil)
+        raise ArgumentError unless max_bytes.is_a?(Integer) && max_bytes.positive? && timeout.is_a?(Numeric) && timeout.positive?
+        @trusted_origin = origin(parse(trusted_origin)) if trusted_origin
+        @max_bytes, @timeout, @client = max_bytes, timeout, client
+      end
+
+      def get(url)
+        uri = parse(url)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @timeout
+        Timeout.timeout(@timeout, FetchError) do
+          headers = { "User-Agent" => "OpenBlogSurfaceBot", "Accept-Encoding" => "identity", "Accept" => "*/*" }
+          if @client
+            response = @client.get(uri.to_s, headers: headers)
+            body = response.body.to_s.b
+            raise FetchError if body.bytesize > @max_bytes
+            return Response.new(status: response.status.to_i, headers: normalize_headers(response.headers), body: body, url: uri.to_s)
+          end
+          addresses = Resolv.getaddresses(uri.hostname)
+          raise FetchError if addresses.empty?
+          addresses.each { |address| public_address!(address) } unless origin(uri) == @trusted_origin
+          http = Net::HTTP.new(uri.hostname, uri.port, nil)
+          http.ipaddr = addresses.first
+          http.use_ssl = uri.scheme == "https"
+          http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+          http.verify_hostname = true
+          http.max_retries = 0
+          http.open_timeout = http.read_timeout = http.write_timeout = remaining(deadline)
+          result = nil
+          http.start do |connection|
+            connection.request(Net::HTTP::Get.new(uri.request_uri, headers)) do |response|
+              bytes = +"".b
+              response.read_body do |chunk|
+                raise FetchError if bytes.bytesize + chunk.bytesize > @max_bytes
+                bytes << chunk
+                http.read_timeout = remaining(deadline)
+              end
+              result = Response.new(status: response.code.to_i, headers: normalize_headers(response.to_hash), body: bytes, url: uri.to_s)
+            end
+          end
+          remaining(deadline)
+          result
+        end
+      rescue URI::InvalidURIError, IPAddr::InvalidAddressError, Resolv::ResolvError, SocketError, SystemCallError,
+        IOError, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Timeout::Error, OpenSSL::SSL::SSLError, ArgumentError
+        raise FetchError
+      end
+
+      private
+
+      def parse(value)
+        raise FetchError unless value.is_a?(String)
+        uri = URI.parse(value)
+        raise FetchError unless uri.is_a?(URI::HTTP) && uri.hostname.present? && uri.userinfo.nil? && !uri.hostname.include?("%")
+        uri.fragment = nil
+        uri
+      end
+
+      def origin(uri) = [ uri.scheme.downcase, uri.hostname.downcase, uri.port ]
+
+      def public_address!(value)
+        ip = IPAddr.new(value).native
+        valid = ip.ipv4? ? ImageFetch::NONPUBLIC_V4.none? { |range| range.include?(ip) } :
+          ImageFetch::GLOBAL_V6.include?(ip) && ImageFetch::NONPUBLIC_V6.none? { |range| range.include?(ip) }
+        raise FetchError unless valid
+      end
+
+      def normalize_headers(headers)
+        headers.to_h.to_h { |key, value| [ key.to_s.downcase, Array(value).join(", ") ] }
+      end
+
+      def remaining(deadline)
+        seconds = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        raise FetchError unless seconds.positive?
+        seconds
+      end
+    end
+  end
+end