proctolog 0.1.0

data/app/models/proctolog/rendering.rb

@@ -0,0 +1,303 @@
+# frozen_string_literal: true
+
+module Proctolog
+  # A description as GitHub rendered it, made ready for the desk's page: cut
+  # into sections at each <h2>, its pictures served from this Mac, its links to
+  # pull requests of the burst pointing at the desk, and the owner's notes
+  # drawn where they belong.
+  #
+  # The words of a section are those of GitHub's nodes alone: a note quotes
+  # `section.text`, and the page's script measures the same text in the browser.
+  # What the desk adds holds no words of a section: marks and boxes on pictures
+  # have no text of their own, and an element it puts in (a "+", a form, the
+  # notes under a box) carries `data-desk`, which both sides skip.
+  class Rendering
+    # A picture's lasting identity is the UUID inside its signed address.
+    UPLOAD = %r{/\d+-(?<asset>\h{8}(?:-\h{4}){3}-\h{12})\.(?<extension>\w+)\?}
+
+    # A picture GitHub never held, an unopened pull request's, at the address
+    # the desk already serves it from.
+    SERVED = %r{\A/pictures/(?<asset>\h{8}(?:-\h{4}){3}-\h{12})\.(?<extension>\w+)\z}
+
+    # Each picture an HTML shows, by its id and its kind.
+    def self.pictures(html) = html.to_s.scan(/src="([^"]*)"/).flatten.filter_map { (it.match(UPLOAD) || it.match(SERVED))&.captures }.uniq
+
+    # A half of a before/after pair, by its picture's name.
+    PAIR = /\A(?<name>.+)-(?<half>before|after)\z/
+
+    # One part of a description: what stands under an <h2>, or before the first.
+    class Section
+      attr_reader :heading, :body, :notes
+
+      def initialize(heading, body)
+        @heading, @body, @notes = heading, body, []
+      end
+
+      def text = nodes.map(&:content).join
+
+      def words = text.split.size
+
+      # Each picture by itself: one alone, or each half of a pair.
+      def figures = body.css("figure.picture:not(.pair), figure.pair > .half")
+
+      def open_decisions = body.css("li.task-list-item").count { !it.at_css("input[checked]") }
+
+      # The boxes of a task list, not the lists inside them.
+      def boxes = body.css("li.task-list-item").reject { it.ancestors("li").any? }
+
+      # The decisions as the desk shows them: each box numbered, and the
+      # non-blocking ones folded under their heading.
+      def lay_out_decisions
+        number_boxes
+        fold_non_blocking
+      end
+
+      # Named as Description#decisions numbers them, so that a link can point at one.
+      def number_boxes = boxes.each_with_index { |box, position| box["id"] = "decision-#{position}" }
+
+      # Adds at the end of each box what the block answers for its place, as
+      # the desk's own: the words of the section stay as they were.
+      def furnish_boxes = boxes.each_with_index { |box, position| box.add_child(yield(position).to_s.strip) }
+
+      # The non-blocking decisions folded away under their own heading:
+      # GitHub's "Non-blocking" heading becomes the fold's summary, and what
+      # follows it in the section goes inside. The words of the section, and
+      # their order, stay as they were; the count beside the heading is the
+      # desk's.
+      def fold_non_blocking
+        heading = body.element_children.find { it.name == "h3" && it.text.strip.casecmp?("Non-blocking") } or return
+        fold = element("details", "class" => "non-blocking", "data-non-blocking" => "")
+        summary = element("summary", {})
+        heading.children.each { summary << it }
+        summary << element("span", "class" => "count", "data-desk" => "", "data-non-blocking-count" => "")
+        heading.xpath("following-sibling::node()").each { fold << it }
+        fold.prepend_child(summary)
+        heading.replace(fold)
+      end
+
+      def noted? = notes.any?(&:live?)
+
+      def html = body.to_html
+
+      # Wraps the slice of each text node that a range of the text covers, so
+      # that words crossing <strong> or <code> are marked in several pieces.
+      def mark(range, note)
+        pieces = slices(range).map { |node, slice| wrap(node, slice, note) }
+        return false if pieces.empty?
+
+        pieces.first["id"] = "note-#{note.id}"
+        notes << note
+        true
+      end
+
+      # A box on a picture, in percent of its size; a box with no width and no
+      # height is a pin.
+      def draw(figure, note, box)
+        x, y, width, height = box.map { (it * 100).round(2) }
+        pin = width.zero? && height.zero?
+        style = {left: x, top: y, width:, height:}.then { pin ? it.slice(:left, :top) : it }.map { |property, percent| "#{property}: #{percent}%" }.join("; ")
+        figure.add_child(element("span", tags(note).merge("class" => pin ? "region pin" : "region", "id" => "note-#{note.id}", "style" => style)))
+        notes << note
+      end
+
+      private
+
+      def document = body.document
+
+      def element(name, attributes) = document.create_element(name, attributes)
+
+      def tags(note) = {"data-note" => note.id, "data-kind" => note.kind, "data-state" => note.state}
+
+      # The text nodes that are GitHub's, in order.
+      def nodes = body.xpath(".//text()[not(ancestor::*[@data-desk])]")
+
+      def slices(range)
+        at = 0
+        nodes.filter_map do |node|
+          from, to = at, (at += node.content.length)
+          inside = ([range.begin, from].max - from)...([range.end, to].min - from)
+          [node, inside] if inside.size.positive? && node.content[inside].present?
+        end
+      end
+
+      def wrap(node, slice, note)
+        text = node.content
+        element("mark", tags(note).merge("class" => "note")).tap do |mark|
+          mark.add_child(document.create_text_node(text[slice]))
+          node.replace(mark)
+          mark.add_previous_sibling(document.create_text_node(text[...slice.begin])) unless slice.begin.zero?
+          mark.add_next_sibling(document.create_text_node(text[slice.end..])) unless slice.end == text.length
+        end
+      end
+    end
+
+    # What was drawn and what was not: `margin` holds the notes found a place,
+    # in the order of the page; `about` those that are about the whole pull
+    # request; `lost` those still waiting on someone whose words or picture are
+    # gone; `set_aside` the rest, withdrawn or confirmed and out of place.
+    attr_reader :sections, :margin, :about, :lost, :set_aside
+
+    # `pages` maps the address of a pull request on GitHub to the desk's page
+    # for it. `linked`, the names of the pictures some line links, folds away
+    # those none does; without it every picture stands.
+    def initialize(html, notes: [], pages: {}, linked: nil)
+      fragment = Nokogiri::HTML5.fragment(html.to_s)
+      @retaken = Set.new
+      serve_pictures_from_here(fragment)
+      pair_pictures(fragment)
+      fold_unlinked(fragment, linked) if linked
+      point_links_at_the_desk(fragment, pages)
+      @sections = sections_of(fragment)
+      lay(notes)
+    end
+
+    # A picture was retaken since the note was drawn on it: only its name is the same.
+    def retaken?(note) = @retaken.include?(note.id)
+
+    private
+
+    def serve_pictures_from_here(fragment)
+      fragment.css("img").each do |img|
+        upload = img["src"].to_s.then { it.match(UPLOAD) || it.match(SERVED) } or next
+        link = img.parent if img.parent.name == "a" && img.parent.element_children.one? && img.parent.text.blank?
+        (link || img).replace(figure_of(fragment.document, img["alt"], **upload.named_captures.symbolize_keys))
+      end
+    end
+
+    def figure_of(document, name, asset:, extension:)
+      document.create_element("figure", "id" => "screenshot-#{name}", "class" => "picture", "data-picture" => name, "data-asset" => asset).tap do |figure|
+        figure.add_child(document.create_element("img", "src" => "/pictures/#{asset}.#{extension}", "alt" => name, "loading" => "lazy", "draggable" => "false"))
+      end
+    end
+
+    # A picture named NAME-before and one named NAME-after are one picture of a
+    # change: a figure where the first stood, both halves side by side, which
+    # a decision links as #screenshot-NAME. Each half keeps its name and its
+    # asset, so a note is drawn on the half it was made on. A half without its
+    # other half, or a name another picture has too, stays as it was.
+    def pair_pictures(fragment)
+      figures = fragment.css("figure.picture")
+      named = figures.group_by { it["data-picture"] }
+      figures.filter_map { it["data-picture"].to_s[PAIR, :name] }.uniq.each do |name|
+        before, after = named["#{name}-before"], named["#{name}-after"]
+        next unless before&.one? && after&.one? && !named.key?(name)
+
+        before.first.replace(pair_of(fragment.document, name, before.first, after.first))
+        after.first.remove
+      end
+    end
+
+    def pair_of(document, name, *halves)
+      document.create_element("figure", "id" => "screenshot-#{name}", "class" => "picture pair", "data-picture" => name).tap do |pair|
+        halves.each do |figure|
+          said = figure["data-picture"][PAIR, :half]
+          half = document.create_element("span", "class" => "half", "id" => figure["id"], "data-half" => said, "data-picture" => figure["data-picture"], "data-asset" => figure["data-asset"])
+          half << figure.at_css("img") << document.create_element("span", said, "class" => "said", "data-desk" => "")
+          pair << half
+        end
+      end
+    end
+
+    # Each run of pictures that nothing links, each with the caption under
+    # it, folds where it stood into one "More pictures (N)", which a reader
+    # opens like a section. A picture a line links stands, and ends a run.
+    def fold_unlinked(fragment, linked)
+      runs = [[]]
+      # A paragraph a picture was parsed out of stays empty, and is no block.
+      fragment.element_children.reject { it.name == "p" && it.children.all?(&:blank?) }.each do |node|
+        if pictures?(node) && !linked?(node, linked) then runs.last << node
+        elsif caption?(node) && pictures?(runs.last.last) then runs.last << node
+        elsif runs.last.any? then runs << []
+        end
+      end
+      runs.reject(&:empty?).each { fold(fragment.document, it) }
+    end
+
+    def fold(document, run)
+      count = run.sum { figures_of(it).size }
+      folded = document.create_element("details", "class" => "more-pictures", "data-more-pictures" => "")
+      folded << (document.create_element("summary") << document.create_element("span", "More pictures (#{count})", "data-desk" => ""))
+      # Everything from its first block to its last, the space between them
+      # too, so the words of the section read as they did.
+      spanned = [run.first].tap { |nodes| nodes << nodes.last.next_sibling until nodes.last == run.last }
+      run.first.add_previous_sibling(folded)
+      spanned.each { folded << it }
+    end
+
+    # A block that is pictures and nothing more: a figure, or a paragraph of them.
+    def pictures?(node) = !node.nil? && (picture?(node) || (node.at_css("figure.picture") && node.xpath(".//text()[not(ancestor::figure)]").all?(&:blank?)))
+
+    # Told by its own name and class: Nokogiri's matches? searches the whole
+    # document for every node it is asked about, and a description has hundreds.
+    def picture?(node) = node.name == "figure" && node.classes.include?("picture")
+
+    # A block's pictures: the block itself when it is one, else those in it.
+    def figures_of(node) = picture?(node) ? [node] : node.css("figure.picture")
+
+    # The words under a picture: a paragraph with no picture in it.
+    def caption?(node) = node.name == "p" && node.at_css("figure.picture").nil?
+
+    # Whether a line links one of its pictures, or a half of one of its pairs.
+    def linked?(node, linked)
+      figures_of(node).flat_map { [it["data-picture"], *it.css(".half").map { |half| half["data-picture"] }] }.any? { linked.include?(it) }
+    end
+
+    def point_links_at_the_desk(fragment, pages)
+      fragment.css("a[href]").each { |link| link["href"] = pages[link["href"]] if pages.key?(link["href"]) }
+    end
+
+    def sections_of(fragment)
+      fragment.children.slice_before { it.name == "h2" }.filter_map do |nodes|
+        heading = nodes.shift.text.strip if nodes.first.name == "h2"
+        next if heading.nil? && nodes.all?(&:blank?)
+
+        body = fragment.document.create_element("div", "data-quotable" => "", "data-section" => heading.to_s)
+        nodes.each { body << it }
+        Section.new(heading, body).tap { it.lay_out_decisions if heading == Description::DECISIONS }
+      end
+    end
+
+    def lay(notes)
+      active = notes.reject(&:withdrawn?)
+      @about = active.reject { anchored?(it) }
+      drawn, unplaced = active.select { anchored?(it) }.partition { draw(it) }
+      @lost = unplaced.select(&:live?)
+      @set_aside = notes - @about - drawn - @lost
+      @margin = sections.flat_map { it.body.css("[data-note]") }.pluck("data-note").uniq.filter_map { |id| drawn.find { it.id == id } }
+    end
+
+    def anchored?(note) = (note.place in Note::Quote | Note::Region)
+
+    def draw(note)
+      case note.place
+      in Note::Quote => quote then mark_quote(note, quote)
+      in Note::Region => region then draw_region(note, region)
+      end
+    end
+
+    # In the section it was written under; in another only when the words are
+    # distinctive enough to be the evidence themselves, as a short phrase
+    # standing elsewhere is likelier another sentence than the one moved.
+    def mark_quote(note, quote)
+      own, others = sections.partition { it.heading == quote.section }
+      (quote.distinctive? ? own + others : own).any? do |section|
+        range = quote.locate(section.text)
+        range && section.mark(range, note)
+      end
+    end
+
+    # On the picture it was drawn on, or on the one of the same name when
+    # that was retaken, if only one is.
+    def draw_region(note, region)
+      pictures = sections.flat_map { |section| section.figures.map { |figure| [section, figure] } }
+      holder = pictures.find { |_, figure| figure["data-asset"] == region.asset } || retaken(note, pictures.select { |_, figure| figure["data-picture"] == region.picture })
+      holder&.then { |section, figure| section.draw(figure, note, region.box) }
+    end
+
+    # The picture a note was drawn on is gone: its name says which one is
+    # meant only when no other picture has it.
+    def retaken(note, named)
+      named.first.tap { @retaken << note.id } if named.one?
+    end
+  end
+end

data/app/models/proctolog/reports.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Proctolog
+  # What agents wrote back about a hand-over: YAML files under reports/, each
+  # a list of entries naming a note, the commit that holds the change and a
+  # reply. Taking them in moves an open note to "addressed" with the newest
+  # entry that names it, and the owner then confirms it.
+  #
+  # A note the owner reopened carries the answer it was reopened against. The
+  # same answer does not address it a second time: only a different commit or
+  # a different reply does.
+  class Reports
+    NOTE = /\A(?<of>sitting|\d+)-\d+\z/
+
+    Entry = Data.define(:note, :commit, :reply, :report) do
+      def answer = {"commit" => commit, "text" => reply}.compact
+    end
+
+    def initialize(burst)
+      @burst = burst
+    end
+
+    # Answers the notes it moved.
+    def take_in = newest_for_each_note.filter_map { address(it) }
+
+    # Entries that name no note of this burst: a mistyped id, said on a page
+    # rather than dropped.
+    def unmatched = entries.reject { note_of(it) }
+
+    # Reports that could not be read as a list of entries.
+    def unreadable = files.reject { list_in(it) }
+
+    private
+
+    def address(entry)
+      note = note_of(entry)
+      return unless note&.open? && note.reply&.slice("commit", "text") != entry.answer
+
+      marks_of(entry).move(note.id, to: "addressed", reply: entry.answer.merge("report" => entry.report))
+    end
+
+    def newest_for_each_note = entries.group_by(&:note).values.map(&:last)
+
+    # The marks a note's id points into: a pull request's, or the sitting's.
+    def marks_of(entry)
+      NOTE.match(entry.note)&.[](:of)&.then { @burst.marks((it == "sitting") ? it : it.to_i) }
+    end
+
+    def note_of(entry) = marks_of(entry)&.note(entry.note)
+
+    # Oldest report first, and each report's entries in its own order.
+    def entries
+      files.flat_map do |file|
+        (list_in(file) || []).grep(Hash).map { Entry.new(note: it["note"].to_s, commit: it["commit"], reply: it["reply"], report: file.basename.to_s) }
+      end
+    end
+
+    def files = Marks.directory_of(@burst).join("reports").glob("*.yml").sort
+
+    def list_in(file)
+      YAML.safe_load_file(file).then { it if it.is_a?(Array) }
+    rescue Psych::Exception
+      nil
+    end
+  end
+end

data/app/models/proctolog/response_files.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Proctolog
+  class ResponseFiles
+    FORMAT = "proctolog-response/v1"
+    STATUSES = %w[done answered deferred needs_owner blocked].freeze
+
+    def initialize(burst) = @burst = burst
+
+    def folder = Marks.directory_of(@burst).join("responses")
+
+    def all
+      folder.glob("*.json").sort.filter_map do |file|
+        data = JSON.parse(file.read)
+        validate!(data)
+        data
+      rescue JSON::ParserError, Failure => e
+        errors << "#{file.basename}: #{e.message}"
+        nil
+      end
+    end
+
+    def errors = @errors ||= []
+
+    def validate!(data)
+      valid = data.is_a?(Hash) && data["format"] == FORMAT && data["repository"] == @burst.repository && data["review"] == @burst.slug &&
+        %w[id batchId summary createdAt].all? { data[it].is_a?(String) && data[it].present? } && data["results"].is_a?(Array)
+      raise Failure, "Not a #{FORMAT} response for this burst." unless valid
+      Time.iso8601(data.fetch("createdAt"))
+
+      ids = data["results"].map do |result|
+        raise Failure, "A result needs id, status, summary, details, evidence and verification." unless result.is_a?(Hash) &&
+          %w[id summary details].all? { result[it].is_a?(String) } && result["id"].present? && result["summary"].present? &&
+          STATUSES.include?(result["status"]) && result["evidence"].is_a?(Array) && result["verification"].is_a?(Array)
+
+        raise Failure, "Invalid evidence or verification for #{result["id"]}." unless result["evidence"].all? { it.is_a?(Hash) && it["label"].is_a?(String) } &&
+          result["verification"].all? { it.is_a?(Hash) && it["command"].is_a?(String) && %w[passed failed not_run].include?(it["result"]) }
+        raise Failure, "A deferred result needs a backlog reference or proposed task." if result["status"] == "deferred" && (!result["backlog"].is_a?(String) || result["backlog"].blank?)
+        raise Failure, "A result head must be a full commit hash." if result.key?("head") && !result["head"].to_s.match?(/\A\h{40}\z/)
+
+        if result["status"] == "needs_owner"
+          question = result["question"]
+          raise Failure, "A question needs prompt, recommendation and a blocking boolean." unless question.is_a?(Hash) &&
+            %w[prompt recommendation].all? { question[it].is_a?(String) && question[it].present? } && [true, false].include?(question["blocking"])
+        end
+        result["id"]
+      end
+      raise Failure, "The response repeats a result id." unless ids.uniq.size == ids.size
+      raise Failure, "Follow-ups must be a list of suggestions." if data.key?("followups") && (!data["followups"].is_a?(Array) || data["followups"].any? { !it.is_a?(String) })
+      data
+    rescue ArgumentError
+      raise Failure, "The response has an invalid createdAt time."
+    end
+  end
+end