archsight 0.4.2 → 0.5.0

data/lib/archsight/pdf/embeds.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require "digest"
+require_relative "typst"
+require_relative "markdown"
+require_relative "tables"
+require_relative "../view_table"
+require_relative "../requirements"
+require_relative "../user_config"
+require_relative "../helpers/embeds"
+require_relative "../helpers/view_blocks"
+require_relative "../helpers/requirements_blocks"
+require_relative "../helpers/resource_resolver"
+require_relative "../query"
+
+module Archsight
+  module Pdf
+    # The live content of a page, as the data of the moment of the export (as the Confluence export does it): a View
+    # (`![[View/Name]]` or a ```view block) and the requirements of some resources as tables, an Analysis as its result,
+    # any other resource as its description. Nothing here raises: what cannot be shown is a note in the document, so
+    # that nothing disappears silently.
+    class Embeds
+      MAX_ANALYSES = 5 # per document, an analysis may run for as long as its own timeout
+      ANALYSIS_TTL = 60 # seconds a result is reused (the PDF endpoint builds the document for every request)
+      ANALYSIS_ENTRIES = 32
+      ANALYSIS_HEADING_OFFSET = 2
+
+      @results = {}
+      @lock = Mutex.new
+
+      class << self
+        # The result of an analysis, from a run of the last minute if there is one
+        def result(database, analysis, now: Time.now)
+          key = [analysis.name, Digest::SHA256.hexdigest(analysis.annotations.to_a.sort.inspect)]
+          entry = @lock.synchronize { @results[key] }
+          return entry[:result] if entry && now - entry[:at] < ANALYSIS_TTL
+
+          require_relative "../analysis"
+          result = Archsight::Analysis::Executor.new(database).execute(analysis)
+          @lock.synchronize do
+            @results[key] = { result: result, at: now }
+            @results.shift while @results.size > ANALYSIS_ENTRIES
+          end
+          result
+        end
+      end
+
+      # @param markdown [#call] `(Boolean inline, String) -> Typst`: renders markdown the way the document does
+      def initialize(database, markdown:, max_analyses: nil)
+        @database = database
+        @markdown = markdown
+        @max_analyses = (max_analyses || UserConfig.setting("pdf", "max_analyses") || MAX_ANALYSES).to_i
+        @analyses = 0
+        @resolver = Helpers::ResourceResolver.new(database)
+      end
+
+      def view_block(source)
+        table(ViewTable.from_block(@database, source))
+      rescue Helpers::ViewBlocks::Error, Archsight::Query::QueryError => e
+        problem("view block: #{e.message}")
+      end
+
+      def requirements_block(source)
+        table(Requirements.table(@database, Helpers::RequirementsBlocks.parse(source)), empty: "No requirements")
+      rescue Helpers::RequirementsBlocks::Error, Archsight::Query::QueryError => e
+        problem("requirements block: #{e.message}")
+      end
+
+      # @param reference [String] `Kind/Name` of `![[Kind/Name]]`
+      def embed(reference)
+        found = @resolver.find(reference)
+        return problem("#{reference}: not found") if found == :missing
+        return problem("#{reference}: ambiguous, write Kind/Name") if found == :ambiguous
+
+        kind, name = found
+        instance = @database.instance_by_kind(kind, name)
+        case kind
+        when "View" then view(instance)
+        when "Analysis" then analysis(instance)
+        else description(kind, instance, reference)
+        end
+      end
+
+      private
+
+      def table(table, empty: "No results")
+        Tables.typst(table, markdown: ->(text) { @markdown.call(true, text) }, empty: empty)
+      end
+
+      def problem(message)
+        Typst.note(message, fill: "#fde8e8")
+      end
+
+      def view(instance)
+        table(ViewTable.from_view(@database, instance))
+      rescue Helpers::ViewBlocks::Error, Archsight::Query::QueryError => e
+        problem("view #{instance.name}: #{e.message}")
+      end
+
+      # A page or menu is not content, a resource without description has nothing to show: its name stands there
+      def description(kind, instance, reference)
+        text = instance.annotations["architecture/description"].to_s.strip
+        return Typst.str(reference) unless Helpers::Embeds.embeddable_kind?(kind) && !text.empty?
+
+        @markdown.call(false, text)
+      end
+
+      def analysis(instance)
+        return problem("Analysis #{instance.name} not run: at most #{@max_analyses} analyses per document") if @analyses >= @max_analyses
+
+        @analyses += 1
+        result = self.class.result(@database, instance)
+        renderer = Archsight::Analysis::TypstRenderer.new([result], full: true)
+        parts = ["#strong[#{Typst.str(instance.name)}]"]
+        parts << renderer.error(result) if result.failed?
+        parts.concat(result.sections.filter_map { |section| analysis_section(renderer, section) })
+        parts.join("\n\n")
+      end
+
+      def analysis_section(renderer, section)
+        case section[:type]
+        when :table then table(analysis_table(section))
+        when :text then @markdown.call(false, section[:content].to_s)
+        else renderer.section(section, heading_offset: ANALYSIS_HEADING_OFFSET)
+        end
+      end
+
+      def analysis_table(section)
+        rows = section[:rows].map { |row| section[:headers].each_index.map { |i| ViewTable::Cell.new(text: row[i].to_s, as: :markdown) } }
+        ViewTable::Table.new(title: "", columns: section[:headers], rows: rows, total: rows.size)
+      end
+    end
+  end
+end

data/lib/archsight/pdf/images.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "open3"
+require_relative "typst"
+require_relative "remote_image"
+require_relative "../assets"
+require_relative "../diagram"
+require_relative "../helpers/diagram_blocks"
+
+module Archsight
+  module Pdf
+    # The pictures of a document: images next to the markdown and `asd` diagrams, copied (or converted) into the
+    # directory typst compiles in. Only what Archsight::Assets serves is ever read.
+    class Images
+      RASTER = %w[.png .jpg .jpeg .gif].freeze
+
+      # @return [Hash{String => String}] staged file name => content
+      attr_reader :files
+
+      # @param base_dir [String, nil] directory of the markdown file, relative to the resources directory
+      # @param pdf_images [Boolean] whether typst can place PDF files; else vector graphics stay SVG
+      def initialize(base_dir:, resources_dir:, resolver: nil, includes: nil, pdf_images: true, remote: RemoteImage.method(:fetch))
+        @base_dir = base_dir.to_s
+        @resources_dir = resources_dir
+        @resolver = resolver
+        @includes = includes
+        @pdf_images = pdf_images
+        @remote = remote && RemoteImage.enabled? ? remote : nil
+        @files = {}
+      end
+
+      # Hook for Markdown: Typst markup of the image a markdown reference names, nil if it cannot be shown
+      def image(src, _alt = nil)
+        return remote(src) if Assets.external?(src)
+
+        path = Assets.resolve_reference(src, @base_dir)
+        file = path && Assets.file_for(path, resources_dir: @resources_dir)
+        return nil unless file && File.size(file) <= Assets::MAX_BYTES
+
+        case File.extname(file).downcase
+        when *RASTER then place(File.binread(file), File.extname(file).downcase)
+        when ".svg" then vector(File.read(file, encoding: "UTF-8"))
+        when ".asd" then diagram(File.read(file, encoding: "UTF-8"))
+        when Assets::DRAWIO then Typst.note("draw.io diagram, shown in Archsight", fill: "#f2f2f2")
+        end
+      end
+
+      # Hook for Markdown: Typst markup of an `asd` diagram, or a note and the source if it does not render
+      def diagram(source)
+        vector(Helpers::DiagramBlocks.standalone_svg(source, resolver: @resolver, includes: @includes))
+      rescue Archsight::Diagram::Error => e
+        note = Typst.note("Diagram could not be rendered: #{e.message}", fill: "#fde8e8")
+        [note, Typst.raw(source, "asd")].join("\n\n")
+      end
+
+      # Vector graphics are placed as PDF (rsvg-convert paints text outlines correctly, typst's own SVG renderer
+      # does not honour `paint-order`), as SVG when that tool or PDF images are not available.
+      def vector(svg, max_height: nil)
+        pdf = @pdf_images ? Images.svg_to_pdf(svg) : nil
+        pdf ? place(pdf, ".pdf", max_height: max_height) : place(svg, ".svg", max_height: max_height)
+      end
+
+      # A PDF page that is already a picture (drawn by another tool), placed as it is
+      def pdf(bytes, max_height: nil)
+        place(bytes, ".pdf", max_height: max_height)
+      end
+
+      # @return [String, nil] PDF bytes of an SVG document, nil without rsvg-convert
+      def self.svg_to_pdf(svg)
+        out, status = Open3.capture2("rsvg-convert", "--format", "pdf", stdin_data: svg, binmode: true)
+        status.success? && !out.empty? ? out : nil
+      rescue Errno::ENOENT
+        nil
+      end
+
+      # An image by URL: fetched when allowed, else the caller shows the alt text
+      def remote(src)
+        return nil unless @remote && src.match?(%r{\Ahttps?://}i)
+
+        bytes, ext = @remote.call(src)
+        place(bytes, ext) if bytes
+      end
+
+      private
+
+      # @param max_height [Float, nil] share of the page height the picture may take (the layout's default if nil)
+      def place(bytes, ext, max_height: nil)
+        name = "img-#{@files.size + 1}#{ext}"
+        @files[name] = bytes
+        "#fit-image(#{Typst.lit(name)}#{", max-height: #{max_height}" if max_height})"
+      end
+    end
+  end
+end

data/lib/archsight/pdf/markdown.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require_relative "../markdown"
+require_relative "../helpers/macros"
+require_relative "../jira_settings"
+require_relative "typst"
+
+module Archsight
+  module Pdf
+    # Markdown (the dialect of Archsight::Markdown) as Typst markup.
+    #
+    # What a document refers to is resolved by the caller through hooks, so this class knows neither the database
+    # nor the file system:
+    #
+    # - `image`:   `(src, alt) -> String | nil`, Typst markup for an embedded image (nil: left out, the alt text is shown)
+    # - `diagram`: `(source) -> String`, Typst markup for an `asd` code block (default: the source as code)
+    # - `label`:   `(target) -> String`, the text of a `[[Target]]` link without label (default: the target)
+    # - `embeds`:  an object with `view_block(source)`, `requirements_block(source)` and `embed(reference)`, each
+    #   returning Typst markup (never raising: what cannot be shown is a note). It makes the live content of a page, a
+    #   ```view or ```requirements block and `![[Kind/Name]]` on a line of its own, part of the document. Without it
+    #   those blocks are shown as written and an embed is its label.
+    class Markdown
+      include Typst
+
+      SYMBOLS = {
+        ndash: "–", mdash: "—", hellip: "…", laquo: "«", raquo: "»", laquo_space: "« ", raquo_space: " »",
+        lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”"
+      }.freeze
+      STATUS_FILL = {
+        "green" => "#d9f2e0", "yellow" => "#fff3cc", "red" => "#fde8e8", "blue" => "#ddeeff", "grey" => "#e8e8e8",
+        "gray" => "#e8e8e8", "purple" => "#eadcf8"
+      }.freeze
+      EMBED = /\A!\[\[([^\]|]+)\]\]\z/
+      # `![[Embed]]`, `[[Target|label]]` and the `{macro:arguments}` of the wiki (see Helpers::Macros)
+      INLINE = /(!)?\[\[([^\]|]+)(?:\|([^\]]+))?\]\]|\{([a-z][a-z0-9-]*)(?::([^{}|\n]*))?\}/
+      ALIGNMENT = { left: "left", center: "center", right: "right" }.freeze
+
+      # @param heading_offset [Integer] added to the level of every heading
+      def initialize(heading_offset: 0, image: nil, diagram: nil, label: nil, embeds: nil)
+        @heading_offset = heading_offset
+        @image = image
+        @diagram = diagram
+        @label = label
+        @embeds = embeds
+      end
+
+      # @param inline [Boolean] for a table cell or list item: paragraphs are joined by a line break
+      # @return [String] Typst markup
+      def render(source, inline: false)
+        out = blocks(Archsight::Markdown.document(source).root.children)
+        inline ? out.gsub(/\n\n+/, " #linebreak() ") : out
+      end
+
+      private
+
+      def blocks(elements)
+        elements.filter_map { |node| block(node) }.join("\n\n")
+      end
+
+      def block(node)
+        case node.type
+        when :p then paragraph(node)
+        when :header then "#heading(level: #{[node.options[:level] + @heading_offset, 6].min})[#{inlines(node.children)}]"
+        when :codeblock then code(node.value, node.options[:lang])
+        when :blockquote then "#quote(block: true)[#{blocks(node.children)}]"
+        when :ul then "#list(#{items(node)})"
+        when :ol then "#enum(#{items(node)})"
+        when :hr then "#line(length: 100%, stroke: 0.5pt + luma(200))"
+        when :table then table(node)
+        when :blank, :html_element, :xml_comment then nil
+        else inlines(node.children) unless node.children.empty?
+        end
+      end
+
+      # A paragraph that is only an image is a figure, centred; one that is only embeds (one per line) is their content
+      def paragraph(node)
+        content = node.children.reject { |c| c.type == :text && c.value.strip.empty? }
+        return image(content.first) if content.size == 1 && content.first.type == :img
+
+        embeds = block_embeds(content)
+        return embeds if embeds
+
+        inlines(node.children)
+      end
+
+      def block_embeds(content)
+        return nil unless @embeds && content.size == 1 && content.first.type == :text
+
+        lines = content.first.value.lines.map(&:strip).reject(&:empty?)
+        return nil unless lines.any? && lines.all? { |line| line.match?(EMBED) }
+
+        lines.map { |line| @embeds.embed(line[EMBED, 1].strip) }.join("\n\n")
+      end
+
+      def items(list)
+        list.children.map { |li| "[#{blocks(li.children)}]" }.join(", ")
+      end
+
+      def code(source, lang)
+        case lang.to_s
+        when "asd" then @diagram ? @diagram.call(source) : raw(source, lang)
+        when "view" then @embeds ? @embeds.view_block(source) : raw(source, lang)
+        when "requirements" then @embeds ? @embeds.requirements_block(source) : raw(source, lang)
+        when "mermaid" then [note("Mermaid diagram, shown as a diagram in Archsight", fill: "#f2f2f2"), raw(source, lang)].join("\n\n")
+        else raw(source, lang)
+        end
+      end
+
+      def table(node)
+        head, body = node.children.partition { |section| section.type == :thead }
+        head_rows = head.flat_map(&:children).map { |tr| cells(tr, header: true) }
+        body_rows = body.flat_map(&:children).map { |tr| cells(tr) }
+        width = (head_rows + body_rows).map(&:size).max || 1
+        header = head_rows.empty? ? "" : "table.header(#{head_rows.flatten.join(", ")}), "
+        "#table(columns: #{width}, #{align(node, width)}#{header}#{body_rows.flatten.join(", ")})"
+      end
+
+      def align(node, width)
+        alignment = Array(node.options[:alignment])
+        return "" unless alignment.any? { |a| ALIGNMENT.key?(a) }
+
+        "align: (#{Array.new(width) { |i| ALIGNMENT.fetch(alignment[i], "auto") }.join(", ")},), "
+      end
+
+      # The header cells are semibold, set on the cells (a show rule on the table header does not reach their text)
+      def cells(row, header: false)
+        row.children.map do |td|
+          content = inlines(td.children)
+          header ? "[#text(weight: \"semibold\")[#{content}]]" : "[#{content}]"
+        end
+      end
+
+      def inlines(elements)
+        elements.map { |node| inline(node) }.join
+      end
+
+      def inline(node)
+        case node.type
+        when :text then text(node.value)
+        when :em then "#emph[#{inlines(node.children)}]"
+        when :strong then "#strong[#{inlines(node.children)}]"
+        when :codespan then "#raw(#{lit(node.value)})"
+        when :a then "#link(#{lit(node.attr["href"])})[#{inlines(node.children)}]"
+        when :br then "#linebreak()"
+        when :img then image(node)
+        when :entity then str(node.value.char)
+        when :smart_quote, :typographic_sym then str(SYMBOLS.fetch(node.value, node.value.to_s))
+        when :html_element then html(node)
+        else inlines(node.children)
+        end
+      end
+
+      def html(node)
+        return "#linebreak()" if node.value == "br"
+
+        checkbox(node) || inlines(node.children)
+      end
+
+      def checkbox(node)
+        return unless node.value == "input" && node.attr["type"] == "checkbox"
+
+        str(node.attr.key?("checked") ? "☑ " : "☐ ")
+      end
+
+      def image(node)
+        src = node.attr["src"].to_s
+        alt = node.attr["alt"].to_s
+        @image&.call(src, alt) || "#emph[#{str(alt.empty? ? src : alt)}]"
+      end
+
+      # Plain text, in which wiki links, embeds and the macros of the wiki are replaced
+      def text(value)
+        out = +""
+        pos = 0
+        value.scan(INLINE) do
+          match = Regexp.last_match
+          out << str(value[pos...match.begin(0)]) if match.begin(0) > pos
+          out << (replacement(match) || str(match[0]))
+          pos = match.end(0)
+        end
+        out << str(value[pos..]) if pos < value.length
+        out
+      end
+
+      def replacement(match)
+        match[2] ? link(match) : macro(match[4], match[5])
+      end
+
+      def link(match)
+        label = match[3]&.strip || @label&.call(match[2].strip) || match[2].strip
+        match[1] ? "#emph[#{str(label)}]" : str(label)
+      end
+
+      # A registered macro with valid arguments; nil leaves the text as written. What needs the running application
+      # (`children`, `pagetree`) has no meaning on paper.
+      def macro(name, arguments)
+        handler = Helpers::Macros.handler(name)
+        return nil unless handler
+
+        value = handler.parse(arguments)
+        return nil unless value
+
+        case name
+        when "status" then badge(value.text.upcase, STATUS_FILL.fetch(value.colour.to_s.downcase, "#e8e8e8"))
+        when "jira" then jira(value)
+        when "emoticon" then str(value.char)
+        when "children", "pagetree" then ""
+        end
+      end
+
+      def jira(key)
+        url = JiraSettings.link_for(key)
+        url ? "#link(#{lit(url)})[#{str(key)}]" : "#raw(#{lit(key)})"
+      end
+    end
+  end
+end

data/lib/archsight/pdf/remote_image.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "resolv"
+require "ipaddr"
+require "uri"
+require_relative "../user_config"
+
+module Archsight
+  module Pdf
+    # Images that a page embeds by URL (`![](https://...)`), fetched by the server so they can go into the PDF.
+    #
+    # The URL comes from page content, so the server must not become a way into the network behind it: only http(s) on
+    # the default ports, only hosts that resolve to public addresses (checked for the address that is connected to, and
+    # again for every redirect), a size and time limit, and the answer must really be a png, jpeg, gif or webp image.
+    # Anything else is `nil` and the document shows the alt text. `pdf.remote_images: false`
+    # (`ARCHSIGHT_PDF_REMOTE_IMAGES`) turns fetching off.
+    module RemoteImage
+      MAX_BYTES = 10 * 1024 * 1024
+      TIMEOUT = 10
+      MAX_REDIRECTS = 3
+      CACHE_SECONDS = 3600
+      CACHE_ENTRIES = 64
+      PORTS = { "http" => 80, "https" => 443 }.freeze
+      NON_PUBLIC = %w[
+        0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.0.0.0/24 192.168.0.0/16
+        198.18.0.0/15 224.0.0.0/4 240.0.0.0/4 ::/128 ::1/128 fc00::/7 fe80::/10 ff00::/8
+      ].map { |range| IPAddr.new(range) }.freeze
+      TYPES = { png: ".png", jpeg: ".jpg", gif: ".gif", webp: ".webp" }.freeze
+
+      @cache = {}
+      @lock = Mutex.new
+
+      class << self
+        # @return [Boolean] whether fetching is on
+        def enabled?(env: ENV)
+          !%w[false no 0 off].include?(UserConfig.setting("pdf", "remote_images", env: env).to_s.strip.downcase)
+        end
+
+        # @return [Array(String, String), nil] image bytes and the file extension, nil if it cannot be had
+        def fetch(url, resolver: Resolv, now: Time.now)
+          cached(url, now) { download(url, resolver) }
+        rescue StandardError
+          nil
+        end
+
+        # Public addresses only: not loopback, private, link-local (cloud metadata), multicast, reserved or unspecified
+        def public_address?(address)
+          ip = IPAddr.new(address.to_s)
+          ip = ip.native if ip.ipv4_mapped?
+          NON_PUBLIC.none? { |range| range.include?(ip) }
+        rescue IPAddr::Error
+          false
+        end
+
+        # @return [Symbol, nil] :png, :jpeg, :gif or :webp from the first bytes
+        def image_type(bytes)
+          head = bytes.byteslice(0, 12).to_s.b
+          return :png if head.start_with?("\x89PNG\r\n\x1a\n".b)
+          return :jpeg if head.start_with?("\xFF\xD8\xFF".b)
+          return :gif if head.start_with?("GIF87a", "GIF89a")
+
+          :webp if head.start_with?("RIFF") && head.byteslice(8, 4) == "WEBP"
+        end
+
+        private
+
+        def cached(url, now)
+          entry = @lock.synchronize { @cache[url] }
+          return entry[:value] if entry && now - entry[:at] < CACHE_SECONDS
+
+          value = yield
+          @lock.synchronize do
+            @cache[url] = { value: value, at: now }
+            @cache.shift while @cache.size > CACHE_ENTRIES
+          end
+          value
+        end
+
+        def download(url, resolver, redirects = 0)
+          uri = URI.parse(url)
+          return nil unless PORTS.key?(uri.scheme) && uri.host && uri.port == PORTS[uri.scheme] && uri.userinfo.nil?
+
+          addresses = resolver.getaddresses(uri.host).map(&:to_s)
+          return nil if addresses.empty? || !addresses.all? { |a| public_address?(a) }
+
+          response, body = get(uri, addresses.first)
+          case response
+          when Net::HTTPSuccess then image(body)
+          when Net::HTTPRedirection
+            redirects < MAX_REDIRECTS ? download(URI.join(url, response["location"].to_s).to_s, resolver, redirects + 1) : nil
+          end
+        end
+
+        # Connects to the address that was checked (not to a second DNS answer); the certificate is checked for the name.
+        # @return [Array(Net::HTTPResponse, String), nil] the response and its body, nil if the body is too large
+        def get(uri, address)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.ipaddr = address
+          http.use_ssl = uri.scheme == "https"
+          http.open_timeout = http.read_timeout = http.write_timeout = TIMEOUT
+          http.request(Net::HTTP::Get.new(uri.request_uri, "Accept" => "image/*", "User-Agent" => "archsight-pdf")) do |res|
+            return [res, nil] unless res.is_a?(Net::HTTPSuccess)
+            return nil if res["Content-Length"].to_i > MAX_BYTES
+
+            body = +"".b
+            res.read_body do |chunk|
+              body << chunk
+              return nil if body.bytesize > MAX_BYTES
+            end
+            return [res, body]
+          end
+        end
+
+        def image(bytes)
+          type = image_type(bytes.to_s)
+          type ? [bytes, TYPES.fetch(type)] : nil
+        end
+      end
+    end
+  end
+end