tensaku 0.0.1

data/lib/tensaku/notes.rb

@@ -0,0 +1,204 @@
+require "json"
+require "time"
+
+module Tensaku
+  # Notes are one JSON file each in tmp/tensaku/notes: local, ignored by git, cleared by tmp:clear.
+  #
+  # A note is open until an agent (or you) resolves it with a reply, or sets it to `question` with a
+  # reply asking for more detail. Resolved notes are deleted after Tensaku.config.prune_after.
+  module Notes
+    STATUSES = %w[open question resolved].freeze
+    ID = /\A[a-z0-9]{4,12}\z/
+    TARGET_KINDS = %w[element text translation markdown].freeze
+
+    class NotFound < Error; end
+
+    module_function
+
+    def dir
+      Tensaku.root.join(Tensaku.config.notes_path)
+    end
+
+    def all
+      return [] unless dir.directory?
+
+      dir.glob("*.json").filter_map { read(it) }.sort_by { it["created_at"] }
+    end
+
+    def list(url: nil, status: nil)
+      statuses = Array(status).map(&:to_s)
+      all.select do |note|
+        (url.nil? || note["url"] == url) && (statuses.empty? || statuses.include?(note["status"]))
+      end
+    end
+
+    def find(id)
+      raise NotFound, "No note #{id.inspect}" unless id.to_s.match?(ID)
+
+      read(dir.join("#{id}.json")) or raise NotFound, "No note #{id.inspect}"
+    end
+
+    # attributes come from the overlay: url, comment, targets, and for area notes rect and viewport.
+    def create(attributes)
+      attributes = attributes.to_h.stringify_keys
+      comment = attributes["comment"].to_s.strip
+      raise Error, "A note needs a comment" if comment.empty?
+
+      now = Time.now.utc.iso8601
+      note = {
+        "id" => new_id,
+        "url" => attributes["url"].to_s,
+        "comment" => comment,
+        "targets" => Array(attributes["targets"]).map { target(it) },
+        "rect" => rect(attributes["rect"]),
+        "viewport" => attributes["viewport"]&.to_s,
+        "status" => "open",
+        "reply" => nil,
+        "created_at" => now,
+        "updated_at" => now
+      }.compact_blank.merge("reply" => nil)
+      write(note)
+    end
+
+    def update(id, status: nil, reply: nil, comment: nil)
+      note = find(id)
+      if status
+        raise Error, "Status must be one of #{STATUSES.join(", ")}" unless STATUSES.include?(status.to_s)
+
+        note["status"] = status.to_s
+      end
+      note["reply"] = reply.to_s.strip.presence unless reply.nil?
+      note["comment"] = comment.to_s.strip unless comment.to_s.strip.empty?
+      note["updated_at"] = Time.now.utc.iso8601
+      write(note)
+    end
+
+    def delete(id)
+      find(id)
+      dir.join("#{id}.json").delete
+    end
+
+    def prune!(now: Time.now)
+      cutoff = now - Tensaku.config.prune_after
+      all.each do |note|
+        next unless note["status"] == "resolved" && Time.iso8601(note["updated_at"]) < cutoff
+
+        dir.join("#{note["id"]}.json").delete
+      end
+    end
+
+    # Blocks until a note is created or changed after `since`, or the timeout passes. Returns the open
+    # notes that are new or changed. Polls the directory, which is plenty for one person and an agent.
+    def watch(since:, timeout: 60, interval: 0.5)
+      deadline = monotonic + timeout
+      loop do
+        fresh = all.select { it["status"] == "open" && Time.iso8601(it["updated_at"]) > since }
+        return fresh if fresh.any? || monotonic >= deadline
+
+        sleep interval
+      end
+    end
+
+    def to_markdown(notes)
+      notes = Array(notes)
+      return "No notes." if notes.empty?
+
+      notes.map { note_markdown(it) }.join("\n\n")
+    end
+
+    def note_markdown(note)
+      lines = [ "## Tensaku note #{note["id"]} (#{note["status"]}) on #{note["url"]}", "", note["comment"] ]
+      targets = Array(note["targets"])
+      if targets.any?
+        lines << ""
+        targets.each { lines << "- #{target_markdown(it)}" }
+      end
+      if (rect = note["rect"])
+        lines << "" << "Area: #{rect["width"]}x#{rect["height"]} at (#{rect["x"]}, #{rect["y"]}) in a #{note["viewport"] || "unknown"} viewport"
+      end
+      lines << "" << "Reply: #{note["reply"]}" if note["reply"]
+      lines.join("\n")
+    end
+
+    def target_markdown(target)
+      location = [ target["file"], target["range"] ].compact.join(":")
+      parts = [ target["kind"] ]
+      parts << "`#{target["tag"]}`" if target["tag"]
+      parts << "`#{target["key"]}` (#{target["locale"]})" if target["key"]
+      parts << "at `#{location}`" unless location.empty?
+      parts << "\"#{target["text"].to_s.squish.truncate(120)}\"" if target["text"].present?
+      context = target["context"].to_h.compact_blank
+      parts << "(#{context.map { |key, value| "#{key}: #{Array(value).join(" < ")}" }.join(", ")})" if context.any?
+      parts.join(" ")
+    end
+
+    # Fills in file positions from what the overlay saw: a Herb stamp, a translation key or a Markdown
+    # range. A target whose source can't be read keeps what the page said about it.
+    def target(raw)
+      raw = raw.to_h.stringify_keys
+      kind = TARGET_KINDS.include?(raw["kind"]) ? raw["kind"] : "element"
+      target = { "kind" => kind, "text" => raw["text"]&.to_s&.truncate(500), "tag" => raw["tag"]&.to_s }
+
+      if raw["stamp"].present?
+        target["stamp"] = raw["stamp"].to_s
+        path, line, column = Source::Template.parse_stamp(raw["stamp"])
+        target["file"] = path
+        target["range"] = "#{line}:#{column}"
+        begin
+          target.merge!(Source::Template.new(path).element_range(line, column).stringify_keys)
+        rescue Error
+          nil
+        end
+      end
+
+      if kind == "translation"
+        target["key"] = raw["key"].to_s
+        target["locale"] = raw["locale"].to_s
+        if (entry = Source::Locale.find(target["locale"], target["key"]))
+          target["file"] = entry[:file]
+          target["range"] = entry[:line].to_s
+        end
+      end
+
+      if kind == "markdown" && raw["md"].present?
+        target["md"] = raw["md"].to_s
+        target["file"], target["range"] = raw["md"].to_s.split(/:(?=\d+:\d+-)/, 2)
+      end
+
+      target["context"] = raw["context"].to_h.slice("partials", "controller", "frame").transform_values { Array(it).map(&:to_s).first(10) }.compact_blank
+      target.compact_blank
+    end
+
+    def rect(raw)
+      return nil unless raw
+
+      raw.to_h.stringify_keys.slice("x", "y", "width", "height").transform_values { it.to_f.round }
+    end
+
+    def new_id
+      loop do
+        id = SecureRandom.alphanumeric(4).downcase
+        return id unless dir.join("#{id}.json").exist?
+      end
+    end
+
+    def read(path)
+      JSON.parse(File.read(path))
+    rescue Errno::ENOENT, JSON::ParserError
+      nil
+    end
+
+    def write(note)
+      dir.mkpath
+      path = dir.join("#{note["id"]}.json")
+      temp = dir.join(".#{note["id"]}.json.tmp")
+      temp.write(JSON.pretty_generate(note) + "\n")
+      File.rename(temp, path)
+      note
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/tensaku/railtie.rb

@@ -0,0 +1,36 @@
+module Tensaku
+  # Everything here is development only. In any other environment the railtie adds nothing, and
+  # neither ReActionView nor Herb is loaded, so templates keep rendering with Erubi.
+  class Railtie < Rails::Railtie
+    if Rails.env.development?
+      require "reactionview"
+      require "herb/engine/visitors/source_attribution_visitor"
+
+      # Before the app's initializers, so config/initializers/reactionview.rb can still adjust things.
+      initializer "tensaku.reactionview", before: :load_config_initializers do
+        ReActionView.configure do |config|
+          config.intercept_erb = true
+          config.engine.visitors.use(Herb::Engine::SourceAttributionVisitor.new)
+        end
+      end
+
+      initializer "tensaku.middleware" do |app|
+        app.middleware.insert_before 0, Tensaku::Middleware
+      end
+
+      initializer "tensaku.i18n" do
+        ActiveSupport.on_load(:action_controller_base) do
+          include Tensaku::I18nMarkers::Controller
+        end
+      end
+
+      config.after_initialize do
+        I18n.backend.extend(Tensaku::I18nMarkers::Backend) unless I18n.backend.is_a?(Tensaku::I18nMarkers::Backend)
+
+        if (warning = Tensaku::Caches.fragment_cache_warning)
+          Rails.logger.warn("[Tensaku] #{warning}")
+        end
+      end
+    end
+  end
+end

data/lib/tensaku/source/html_text.rb

@@ -0,0 +1,121 @@
+module Tensaku
+  module Source
+    # Static HTML text as the browser shows it, mapped back to its source bytes.
+    #
+    # The source is cut into segments: one per character, one per entity (`&`), and one per run
+    # of whitespace (shown as a single space, or nothing at the edges of an element). An edit keeps the
+    # source of every segment the edit didn't touch, so `©` or a line break in the middle of a
+    # paragraph survives an edit elsewhere in the same text.
+    class HtmlText
+      Segment = Data.define(:raw, :text)
+
+      ENTITIES = {
+        "amp" => "&", "lt" => "<", "gt" => ">", "quot" => '"', "apos" => "'", "nbsp" => "\u00A0",
+        "copy" => "\u00A9", "reg" => "\u00AE", "trade" => "\u2122", "hellip" => "\u2026",
+        "mdash" => "\u2014", "ndash" => "\u2013", "lsquo" => "\u2018", "rsquo" => "\u2019",
+        "ldquo" => "\u201C", "rdquo" => "\u201D", "laquo" => "\u00AB", "raquo" => "\u00BB",
+        "middot" => "\u00B7", "bull" => "\u2022", "euro" => "\u20AC", "pound" => "\u00A3",
+        "times" => "\u00D7", "divide" => "\u00F7", "deg" => "\u00B0", "shy" => "\u00AD",
+        "iexcl" => "\u00A1", "iquest" => "\u00BF", "larr" => "\u2190", "rarr" => "\u2192"
+      }.freeze
+
+      WHITESPACE = /\A[ \t\r\n\f]+\z/
+      TOKEN = /&(?:#[xX]\h+|#\d+|[a-zA-Z][a-zA-Z0-9]*);|[ \t\r\n\f]+|./m
+
+      attr_reader :segments
+
+      # collapse: whitespace runs read as one space (HTML text), or stay as written (attribute values).
+      # trim_start/trim_end: a whitespace run at that edge reads as nothing, as it does at the start or
+      # end of a block element.
+      def initialize(raw, collapse: true, trim_start: false, trim_end: false)
+        @raw = raw
+        tokens = raw.scan(TOKEN)
+        @segments = tokens.each_with_index.map do |token, index|
+          edge = (trim_start && index.zero?) || (trim_end && index == tokens.size - 1)
+          Segment.new(token, decode(token, collapse:, edge:))
+        end
+      end
+
+      def text
+        segments.map(&:text).join
+      end
+
+      # The new source for this text after an edit, changing only the segments the edit touched.
+      def rewrite(new_text, escape:)
+        old_text = text
+        return @raw if new_text == old_text
+
+        prefix = common_prefix(old_text, new_text)
+        suffix = common_suffix(old_text, new_text, limit: [ old_text.length, new_text.length ].min - prefix)
+
+        head = take_segments(segments, prefix)
+        tail = take_segments(segments.reverse, suffix).reverse
+        head_length = head.sum { it.text.length }
+        tail_length = tail.sum { it.text.length }
+        middle = new_text[head_length...(new_text.length - tail_length)]
+
+        head.map(&:raw).join + escape.call(middle) + tail.map(&:raw).join
+      end
+
+      def self.escape_text(text)
+        text.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;").gsub("\u00A0", "&nbsp;")
+      end
+
+      def self.attribute_escaper(quote)
+        lambda do |text|
+          escaped = text.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;")
+          quote == "'" ? escaped.gsub("'", "&#39;") : escaped.gsub('"', "&quot;")
+        end
+      end
+
+      private
+
+      def decode(token, collapse:, edge:)
+        if token.match?(WHITESPACE)
+          return token unless collapse
+
+          edge ? "" : " "
+        elsif token.start_with?("&") && token.length > 1
+          decode_entity(token)
+        else
+          token
+        end
+      end
+
+      def decode_entity(token)
+        name = token[1...-1]
+        code = if name.start_with?("#x", "#X") then name[2..].to_i(16)
+        elsif name.start_with?("#") then name[1..].to_i
+        end
+        return [ code ].pack("U") if code&.between?(1, 0x10FFFF)
+
+        ENTITIES.fetch(name, token)
+      end
+
+      def common_prefix(a, b)
+        length = 0
+        length += 1 while length < a.length && length < b.length && a[length] == b[length]
+        length
+      end
+
+      def common_suffix(a, b, limit:)
+        length = 0
+        length += 1 while length < limit && a[-1 - length] == b[-1 - length]
+        length
+      end
+
+      # The leading segments whose text fits entirely within `length` characters.
+      def take_segments(list, length)
+        taken = []
+        used = 0
+        list.each do |segment|
+          break if used + segment.text.length > length
+
+          used += segment.text.length
+          taken << segment
+        end
+        taken
+      end
+    end
+  end
+end

data/lib/tensaku/source/locale.rb

@@ -0,0 +1,158 @@
+require "psych"
+
+module Tensaku
+  module Source
+    # Locale YAML, edited without load-and-dump: Psych's parse tree gives each scalar's position, and
+    # only the value's bytes change. Comments, ordering and quoting stay as they were. Every write is
+    # checked by loading the result and comparing the edited value before the file is replaced.
+    module Locale
+      EXTENSIONS = %w[.yml .yaml].freeze
+      INTERPOLATION = /%\{[^}]+\}/
+
+      Entry = Data.define(:file, :path, :source, :node, :value) do
+        def line = node.start_line + 1
+
+        def to_h
+          { file:, line:, value:, style: Locale.style_name(node.style), editable: true }
+        end
+      end
+
+      module_function
+
+      # The file that defines locale.key, the last one in I18n.load_path order (the one that wins).
+      # Only files inside Tensaku's writable paths count.
+      def find(locale, key)
+        candidates(locale).reverse_each do |relative, path|
+          source = File.read(path, mode: "rb").force_encoding(Encoding::UTF_8)
+          node = scalar_at(Psych.parse(source), [ locale.to_s, *key.to_s.split(".") ])
+          return Entry.new(relative, path, source, node, node.value) if node
+        rescue Psych::SyntaxError
+          next
+        end
+        nil
+      end
+
+      # Every available locale's entry for the key, for the other-locales view.
+      def lookup_all(key)
+        I18n.available_locales.to_h do |locale|
+          entry = find(locale, key)
+          [ locale.to_s, entry&.to_h || { value: I18n.t(key, locale:, default: nil)&.then { it.is_a?(String) ? it : nil }, editable: false } ]
+        end
+      end
+
+      # Replaces the value of locale.key. `original` is the value the page was rendered with.
+      def edit(locale, key, original:, text:)
+        text = text.to_s.delete("\r")
+        entry = find(locale, key) or raise Error, "#{locale}.#{key} isn't defined in a writable locale file"
+        raise StaleSource, "#{entry.file} changed since the page was rendered. Reload the page and try again." unless entry.value == original.to_s
+
+        if text.scan(INTERPOLATION).sort != entry.value.scan(INTERPOLATION).sort
+          raise Error, "Keep the same %{...} placeholders: #{entry.value.scan(INTERPOLATION).join(", ")}"
+        end
+        return { changed: false, file: entry.file } if text == entry.value
+
+        start, finish = byte_range(entry.source, entry.node)
+        replacement = render(entry, text, entry.source.byteslice(start...finish))
+        updated = FileWrite.splice(entry.source, [ { start:, end: finish, text: replacement } ])
+
+        check!(updated, locale, key, text, entry.file)
+        FileWrite.atomic(entry.path, updated)
+        { changed: true, file: entry.file, line: entry.line }
+      end
+
+      def style_name(style)
+        { Psych::Nodes::Scalar::PLAIN => "plain", Psych::Nodes::Scalar::SINGLE_QUOTED => "single",
+          Psych::Nodes::Scalar::DOUBLE_QUOTED => "double", Psych::Nodes::Scalar::LITERAL => "literal",
+          Psych::Nodes::Scalar::FOLDED => "folded" }.fetch(style, "plain")
+      end
+
+      def candidates(locale)
+        allowed = Guard.writable_dirs.map(&:to_s)
+        root = Tensaku.root.realpath
+        I18n.load_path.flatten.filter_map do |entry|
+          path = Pathname(entry.to_s).expand_path
+          next unless EXTENSIONS.include?(path.extname) && path.file?
+
+          path = path.realpath
+          next unless allowed.any? { path.to_s.start_with?("#{it}/") }
+
+          [ path.relative_path_from(root).to_s, path ]
+        end.uniq.select { |_, path| File.read(path).include?(locale.to_s) }
+      end
+
+      def scalar_at(document, keys)
+        node = document&.root
+        keys.each do |key|
+          return nil unless node.is_a?(Psych::Nodes::Mapping)
+
+          pair = node.children.each_slice(2).to_a.reverse.find { |name, _| name.is_a?(Psych::Nodes::Scalar) && name.value == key }
+          return nil unless pair
+
+          node = pair.last
+        end
+        node if node.is_a?(Psych::Nodes::Scalar)
+      end
+
+      # Psych positions are 0-based lines and character columns.
+      def byte_range(source, node)
+        lines = source.lines
+        offset = ->(line, column) { lines.first(line).sum(&:bytesize) + lines.fetch(line, "")[0, column].to_s.bytesize }
+        [ offset.call(node.start_line, node.start_column), offset.call(node.end_line, node.end_column) ]
+      end
+
+      # The new scalar in the style the old one had. A plain scalar that would read differently as
+      # plain YAML (a leading quote, `: `, ` #`, `true`, a number) is written double-quoted instead.
+      def render(entry, text, old)
+        case entry.node.style
+        when Psych::Nodes::Scalar::SINGLE_QUOTED
+          text.include?("\n") ? double_quoted(text) : "'#{text.gsub("'", "''")}'"
+        when Psych::Nodes::Scalar::DOUBLE_QUOTED
+          double_quoted(text)
+        when Psych::Nodes::Scalar::LITERAL, Psych::Nodes::Scalar::FOLDED
+          block(entry, text, old)
+        else
+          plain_safe?(text) ? text : double_quoted(text)
+        end
+      end
+
+      def double_quoted(text)
+        escaped = text.gsub("\\", "\\\\\\\\").gsub('"', '\"').gsub("\n", "\\n").gsub("\t", "\\t")
+        %("#{escaped}")
+      end
+
+      def plain_safe?(text)
+        return false if text.empty? || text.include?("\n") || text != text.strip
+        return false if text.match?(/\A[-?:,\[\]{}#&*!|>'"%@`]/) || text.include?(": ") || text.include?(" #") || text.end_with?(":")
+
+        Psych.safe_load("v: #{text}")["v"] == text
+      rescue Psych::Exception
+        false
+      end
+
+      # Keeps the header (`>-`, `|+`), the indentation and any trailing blank lines of a block scalar.
+      def block(entry, text, old)
+        header, body = old.split("\n", 2)
+        indent = body.to_s[/\A[ ]+/] || "  "
+        trailing = body.to_s[/\n*\z/].to_s
+        lines = if entry.node.style == Psych::Nodes::Scalar::FOLDED
+          text.split("\n", -1)
+        else
+          text.chomp.split("\n", -1)
+        end
+        content = lines.map { it.empty? ? "" : "#{indent}#{it}" }.join("\n")
+        "#{header}\n#{content}#{trailing.empty? ? "" : trailing}"
+      end
+
+      def check!(updated, locale, key, text, file)
+        loaded = Psych.safe_load(updated, aliases: true, permitted_classes: [ Symbol, Date, Time ])
+        value = [ locale.to_s, *key.to_s.split(".") ].reduce(loaded) { |node, name| node.is_a?(Hash) ? node[name] : nil }
+        expected = value.is_a?(String) && value.end_with?("\n") && !text.end_with?("\n") ? value.chomp : value
+        return if expected == text
+
+        raise Error, "Tensaku couldn't write that value to #{file} without changing its meaning. Edit the file directly."
+      rescue Psych::Exception => error
+        raise Error, "The edit would break #{file} (#{error.message}). Nothing was written."
+      end
+    end
+  end
+end

data/lib/tensaku/source/markdown.rb

@@ -0,0 +1,78 @@
+module Tensaku
+  module Source
+    # Markdown blocks, addressed by the ranges a producer writes:
+    # data-tensaku-md="path:startLine:startCol-endLine:endCol", 1-based lines and byte columns, end
+    # column inclusive (commonmarker's sourcepos). Tensaku doesn't render Markdown; it replaces text
+    # inside a block's range and leaves the rest of the file alone.
+    class Markdown
+      EXTENSIONS = %w[.md .markdown].freeze
+
+      # Block syntax at the start of a range that stays locked: heading hashes, list markers, quotes.
+      PREFIX = /\A(?:\#{1,6}[ \t]+|[-*+][ \t]+(?:\[[ xX]\][ \t]+)?|\d{1,9}[.)][ \t]+|>[ \t]?)/
+
+      # Block tags whose text is edited; others (lists, tables, code) only get notes.
+      EDITABLE = %w[p h1 h2 h3 h4 h5 h6 li blockquote].freeze
+
+      def self.parse_range(value)
+        match = value.to_s.match(/\A(?<path>.+):(?<sl>\d+):(?<sc>\d+)-(?<el>\d+):(?<ec>\d+)\z/)
+        raise Error, "Not a Markdown range: #{value.inspect}" unless match
+
+        [ match[:path], *%i[sl sc el ec].map { match[it].to_i } ]
+      end
+
+      attr_reader :relative_path
+
+      def initialize(range)
+        @relative_path, @start_line, @start_column, @end_line, @end_column = self.class.parse_range(range)
+        @path = Guard.writable_file!(@relative_path, extensions: EXTENSIONS)
+        @source = File.read(@path, mode: "rb").force_encoding(Encoding::UTF_8)
+      end
+
+      def describe(tag: nil)
+        raw = block_source
+        prefix = raw[PREFIX].to_s
+        {
+          file: relative_path,
+          range: "#{@start_line}:#{@start_column}-#{@end_line}:#{@end_column}",
+          editable: tag.nil? || EDITABLE.include?(tag.to_s.downcase),
+          prefix:,
+          text: raw.delete_prefix(prefix),
+          original: raw
+        }
+      end
+
+      # original: the block's whole source when the editor opened. text: the new source after the
+      # locked prefix.
+      def edit(original:, text:)
+        @source = File.read(@path, mode: "rb").force_encoding(Encoding::UTF_8)
+        raw = block_source
+        raise StaleSource, "#{relative_path} changed since the page was rendered. Reload the page and try again." unless raw == original.to_s
+
+        replacement = raw[PREFIX].to_s + text.to_s.tr(" ", " ").delete("\r")
+        return { changed: false, file: relative_path } if replacement == raw
+
+        start, finish = byte_range
+        FileWrite.atomic(@path, FileWrite.splice(@source, [ { start:, end: finish, text: replacement } ]))
+        { changed: true, file: relative_path }
+      end
+
+      private
+
+      def block_source
+        start, finish = byte_range
+        @source.byteslice(start...finish).force_encoding(Encoding::UTF_8)
+      end
+
+      def byte_range
+        lines = @source.b.lines
+        raise StaleSource, "#{relative_path} is shorter than the page says. Reload the page." if @end_line > lines.size
+
+        offset = ->(line) { lines.first(line - 1).sum(&:bytesize) }
+        start = offset.call(@start_line) + @start_column - 1
+        last_line = lines[@end_line - 1].chomp
+        finish = offset.call(@end_line) + [ @end_column, last_line.bytesize ].min
+        [ start, finish ]
+      end
+    end
+  end
+end