RubyGems - commenter - Versions diffs - 0.3.0 → 0.3.1 - Mend

commenter 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (21) hide show

checksums.yaml +4 -4
data/.rubocop.yml +4 -0
data/CLAUDE.md +17 -7
data/README.adoc +35 -6
data/lib/commenter/cli.rb +10 -18
data/lib/commenter/comment.rb +2 -10
data/lib/commenter/comment_sheet.rb +8 -0
data/lib/commenter/comment_type.rb +35 -0
data/lib/commenter/github_integration.rb +11 -60
data/lib/commenter/github_session.rb +35 -0
data/lib/commenter/parser/osd_xlsx_parser.rb +1 -10
data/lib/commenter/parser/track_change_docx_parser.rb +332 -0
data/lib/commenter/parser.rb +8 -10
data/lib/commenter/version.rb +1 -1
data/lib/commenter.rb +5 -0
data/spec/commenter/comment_sheet_spec.rb +25 -0
data/spec/commenter/comment_type_spec.rb +45 -0
data/spec/commenter/github_integration_spec.rb +65 -32
data/spec/commenter/track_change_docx_parser_spec.rb +277 -0
data/spec/support/redline_docx_builder.rb +88 -0
metadata +8 -2

data/lib/commenter/parser/track_change_docx_parser.rb ADDED Viewed

@@ -0,0 +1,332 @@
+# frozen_string_literal: true
+require "zip"
+require "nokogiri"
+require_relative "../comment"
+require_relative "../comment_sheet"
+module Commenter
+  class Parser
+    # Extracts tracked changes from a redlined Word document into an ISO
+    # 2012-03 comment sheet.
+    #
+    # Captured track changes (w:ins, w:del, w:moveFrom, w:moveTo):
+    # - proposed_change renders the change itself, e.g. +Insert: "text"+
+    # - locality.clause is resolved from the nearest preceding heading
+    # - locality.element is resolved from caption/inline references
+    #   (Table N, Figure N, Formula (N), NOTE n) within the current clause
+    # - observations can be stamped via the observations or accept_all options
+    #
+    # Reviewer comment threads (word/comments.xml w:comment) are emitted
+    # after the track changes with -CNNN ids: the remark verbatim in
+    # comments, its instruction reworded as the proposed change, and empty
+    # observations for the owner to draft.
+    #
+    # Self-closing markers (paragraph-mark insertions inside rPr) carry no
+    # content and are skipped.
+    #
+    # word/document.xml is streamed with Nokogiri::XML::Reader because redline
+    # documents can exceed 100 MB.
+    class TrackChangeDocxParser
+      W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+      HEADING_STYLE = /\A(?:Heading[1-6]|h[2-5]annex\d*|ANNEX|BaseHeading)\z/i
+      KIND_COMMENT = { "ins" => "insertion", "del" => "deletion", "moveFrom" => "move (from)",
+                       "moveTo" => "move (to)" }.freeze
+      KIND_LABEL = { "ins" => "Insert", "del" => "Delete", "moveFrom" => "Move from",
+                     "moveTo" => "Move to" }.freeze
+      DEFAULT_BODY = "CS"
+      WHOLE_DOCUMENT = "_whole document"
+      ACCEPT_ALL = "Accepted. Tracked change accepted."
+      attr_reader :skipped_markers
+      def parse(path, options = {})
+        body = options[:body] || DEFAULT_BODY
+        changes = []
+        remarks = {}
+        anchors = {}
+        @skipped_markers = 0
+        Zip::File.open(path) do |zip|
+          entry = zip.glob("word/document.xml").first
+          raise Commenter::Error, "word/document.xml not found in #{path}" unless entry
+          remarks = read_remarks(zip)
+          changes = stream_changes(entry.get_input_stream, anchors)
+        end
+        CommentSheet.new(
+          version: "2012-03",
+          date: sheet_date(changes),
+          document: options[:document],
+          stage: options[:stage],
+          comments: build_change_comments(changes, body, options) +
+                    build_remark_comments(remarks, anchors, body, options)
+        )
+      end
+      # Turns a reviewer's remark phrased as a request ("Please remove this
+      # NOTE...") into the corresponding proposed change ("Remove this
+      # NOTE...").
+      def self.reword_remark(text)
+        text.to_s.strip.sub(/\APlease\s+/i, "").sub(/\A[a-z]/, &:upcase)
+      end
+      private
+      def read_remarks(zip)
+        entry = zip.glob("word/comments.xml").first
+        return {} unless entry
+        document = Nokogiri::XML(entry.get_input_stream)
+        document.xpath("//w:comment", "w" => W_NS).each_with_object({}) do |node, remarks|
+          remarks[node["w:id"]] = {
+            author: node["w:author"],
+            date: node["w:date"],
+            text: node.xpath(".//w:t", "w" => W_NS).map(&:text).join
+          }
+        end
+      end
+      def sheet_date(changes)
+        dates = changes.filter_map { |change| change[:date].to_s[/\A\d{4}-\d{2}-\d{2}/] }
+        dates.max
+      end
+      def build_change_comments(changes, body, options)
+        changes.each_with_index.map do |change, index|
+          kind = change[:kind]
+          Comment.new(
+            id: format("%<body>s-%03<index>d", body: body, index: index + 1),
+            body: body,
+            locality: { clause: change[:clause], line_number: nil, element: change[:element] },
+            type: options[:type] || "te",
+            comments: "Track change (#{KIND_COMMENT.fetch(kind, kind)})",
+            proposed_change: "#{KIND_LABEL.fetch(kind, kind)}: \"#{change[:text].strip}\"",
+            observations: observations_for(options)
+          )
+        end
+      end
+      def build_remark_comments(remarks, anchors, body, options)
+        remarks.keys.sort_by(&:to_i).each_with_index.map do |remark_id, index|
+          remark = remarks[remark_id]
+          anchor = anchors[remark_id] || {}
+          clause = anchor[:clause].to_s.empty? ? WHOLE_DOCUMENT : anchor[:clause]
+          Comment.new(
+            id: format("%<body>s-C%03<index>d", body: body, index: index + 1),
+            body: body,
+            locality: { clause: clause, line_number: nil, element: anchor[:element] },
+            type: options[:remark_type] || "ed",
+            comments: remark[:text],
+            proposed_change: self.class.reword_remark(remark[:text]),
+            observations: nil
+          )
+        end
+      end
+      def observations_for(options)
+        return nil if options[:exclude_observations]
+        options[:observations] || (options[:accept_all] ? ACCEPT_ALL : nil)
+      end
+      def stream_changes(io, anchors = {})
+        changes = []
+        change_stack = []
+        paragraph = ParagraphState.new
+        Nokogiri::XML::Reader(io).each do |reader|
+          case reader.node_type
+          when Nokogiri::XML::Reader::TYPE_ELEMENT
+            handle_start(reader, change_stack, paragraph, anchors)
+          when Nokogiri::XML::Reader::TYPE_TEXT, Nokogiri::XML::Reader::TYPE_SIGNIFICANT_WHITESPACE
+            handle_text(reader, change_stack, paragraph)
+          when Nokogiri::XML::Reader::TYPE_END_ELEMENT
+            handle_end(reader, change_stack, paragraph, changes)
+          end
+        end
+        changes
+      end
+      def handle_start(reader, change_stack, paragraph, anchors = {})
+        case reader.local_name
+        when "p"
+          paragraph.begin_paragraph
+        when "pStyle"
+          paragraph.style = reader.attribute("w:val") || reader.attribute("val")
+        when "ins", "del", "moveFrom", "moveTo"
+          start_change(reader, change_stack, paragraph)
+        when "commentRangeStart"
+          anchor_remark(reader, paragraph, anchors)
+        end
+      end
+      def anchor_remark(reader, paragraph, anchors)
+        id = reader.attribute("w:id")
+        anchors[id] = {
+          clause: paragraph.pending_clause || paragraph.clause,
+          element: paragraph.pending_element || paragraph.element
+        }
+      end
+      def start_change(reader, change_stack, paragraph)
+        if reader.empty_element?
+          @skipped_markers += 1
+          return
+        end
+        change_stack.push(
+          kind: reader.local_name,
+          id: reader.attribute("w:id"),
+          author: reader.attribute("w:author"),
+          date: reader.attribute("w:date"),
+          text: +"",
+          clause: paragraph.clause,
+          element: paragraph.element
+        )
+      end
+      def handle_text(reader, change_stack, paragraph)
+        if (current = change_stack.last)
+          current[:text] << reader.value
+        else
+          paragraph.append_text(reader.value)
+        end
+      end
+      def handle_end(reader, change_stack, paragraph, changes)
+        case reader.local_name
+        when "p"
+          paragraph.commit_paragraph
+        when "ins", "del", "moveFrom", "moveTo"
+          change = change_stack.pop
+          changes << finalize_change(change, paragraph) if change
+        end
+      end
+      def finalize_change(change, paragraph)
+        # A change inside a heading or caption paragraph belongs to that
+        # paragraph, which is not committed yet at this point.
+        change[:clause] = paragraph.pending_clause || change[:clause]
+        change[:element] = paragraph.pending_element || change[:element]
+        change[:clause] = WHOLE_DOCUMENT if change[:clause].to_s.empty?
+        change
+      end
+      # Tracks the current paragraph's style and text to resolve the locality
+      # context of a change:
+      #
+      # - clause: the clause number of the most recent heading paragraph; an
+      #   unnumbered sub-heading inherits its parent heading's clause
+      # - element: the nearest Table/Figure/Formula/NOTE reference, scoped to
+      #   the current clause (caption paragraphs and inline mentions both
+      #   count)
+      class ParagraphState
+        NUMBERED_CLAUSE = /\A(?:\d+(?:\.\d+)*|Annex\s+[A-Z](?:\.\d+)*|Bibliography|Foreword|Introduction)\z/i
+        ELEMENT_START = /\A\s*(NOTE\s+\d+|NOTE(?=\s*[—:-])|Table\s+(?:[A-Z]\.)?\d+(?:\.\d+)*|Figure\s+(?:[A-Z]\.)?\d+(?:\.\d+)*)\b/i
+        FORMULA = /\b(Formula\s*\(\d+(?:\.\d+)*\))/
+        ELEMENT_ANY = /\b(Table\s+(?:[A-Z]\.)?\d+(?:\.\d+)*|Figure\s+(?:[A-Z]\.)?\d+(?:\.\d+)*|NOTE\s+\d+)\b/i
+        attr_writer :style
+        attr_reader :clause, :element
+        def initialize
+          @style = nil
+          @text = +""
+          @clause = ""
+          @element = nil
+          @heading_stack = []
+        end
+        def begin_paragraph
+          @style = nil
+          @text = +""
+        end
+        def append_text(value)
+          @text << value
+        end
+        def pending_clause
+          return unless heading_paragraph?
+          clause = self.class.clause_for(@text)
+          clause unless clause.empty?
+        end
+        def pending_element
+          self.class.element_for(@text)
+        end
+        def commit_paragraph
+          if heading_paragraph?
+            commit_heading
+          else
+            @element = self.class.element_for(@text) || @element
+          end
+        end
+        def heading_paragraph?
+          @style&.match?(HEADING_STYLE)
+        end
+        def self.numbered_clause?(clause)
+          clause.to_s.match?(NUMBERED_CLAUSE)
+        end
+        # Extracts the clause identifier from heading text: the leading number
+        # ("4.2.1 Thermodynamic ..." -> "4.2.1"), an annex reference, or a
+        # well-known unnumbered section. Runs may be concatenated without
+        # spaces ("4.2.1Thermodynamic"), so the number is matched with a
+        # lookahead.
+        def self.clause_for(text)
+          s = text.to_s.strip
+          return "" if s.empty?
+          return Regexp.last_match(1) if s =~ /\A\s*(Annex\s+[A-Z](?:\.\d+)*)\b/i
+          return Regexp.last_match(1) if s =~ /\A\s*(Bibliography|Foreword|Introduction)\b/i
+          return Regexp.last_match(1) if s =~ /\A\s*((?:\d+\.)*\d+)(?=[A-Z\s])/
+          return Regexp.last_match(1) if s =~ /\A\s*((?:\d+\.)*\d+)\b/
+          s
+        end
+        # Resolves the element reference of a paragraph: a caption opening the
+        # paragraph ("Table 3 — ...", "NOTE 2 ...", "Formula (9):") wins over an
+        # inline mention ("the values in Table 5 shall ...").
+        def self.element_for(text)
+          s = text.to_s
+          match = s.match(ELEMENT_START) || s.match(FORMULA) || s.match(ELEMENT_ANY)
+          match[1].squeeze(" ").strip if match
+        end
+        private
+        def commit_heading
+          level = heading_level
+          clause = self.class.clause_for(@text)
+          clause = inherited_clause(level) unless self.class.numbered_clause?(clause)
+          @heading_stack.pop while @heading_stack.last && @heading_stack.last[0] >= level
+          @heading_stack << [level, clause]
+          @clause = clause unless clause.empty?
+          # Element references belong to the clause they appear in.
+          @element = nil
+        end
+        def inherited_clause(level)
+          parent = @heading_stack.reverse.find { |(lvl, _)| lvl < level }
+          parent ? parent[1] : ""
+        end
+        def heading_level
+          m = @style.match(/\A(?:Heading|h)(\d+)/i)
+          m ? m[1].to_i : 1
+        end
+      end
+    end
+  end
+end

data/lib/commenter/parser.rb CHANGED Viewed

@@ -5,6 +5,7 @@ require "pathname"
 require_relative "comment_sheet"
 require_relative "comment"
 require_relative "parser/osd_xlsx_parser"
+require_relative "parser/track_change_docx_parser"
 module Commenter
   class Parser
@@ -16,6 +17,8 @@ module Commenter
         parse_docx(input_path, options)
       when :xlsx
         parse_xlsx(input_path, options)
+      when :redline
+        parse_redline(input_path, options)
       else
         raise "Unsupported file format: #{input_path}. Supported formats: .docx, .xlsx"
       end
@@ -38,6 +41,10 @@ module Commenter
       OsdXlsxParser.new.parse(xlsx_path, options)
     end
+    def parse_redline(docx_path, options)
+      TrackChangeDocxParser.new.parse(docx_path, options)
+    end
     def parse_docx(docx_path, options)
       doc = Docx::Document.open(docx_path)
@@ -83,7 +90,7 @@ module Commenter
           clause: presence(cells[2]),
           element: presence(cells[3])
         },
-        type: normalize_type(cells[4]),
+        type: CommentType.code(cells[4]),
         comments: cells[5] || "",
         proposed_change: presence(cells[6]),
         user_name: cells[0].to_s.strip
@@ -117,15 +124,6 @@ module Commenter
       value && !value.empty? ? value : nil
     end
-    def normalize_type(type_str)
-      case type_str.to_s.strip.downcase
-      when "editorial" then "ed"
-      when "technical" then "te"
-      when "general" then "ge"
-      else type_str.to_s.strip
-      end
-    end
     def extract_metadata(doc)
       metadata = { date: nil, document: nil, project: nil }

data/lib/commenter/version.rb CHANGED Viewed

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 module Commenter
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

data/lib/commenter.rb CHANGED Viewed

@@ -7,6 +7,11 @@ require_relative "commenter/parser"
 require_relative "commenter/filler"
 require_relative "commenter/github_integration"
+# Commenter converts ISO comment sheets (DOCX/XLSX) to structured YAML and
+# syncs comments to GitHub issues.
 module Commenter
   class Error < StandardError; end
+  autoload :CommentType, "commenter/comment_type"
+  autoload :GitHubSession, "commenter/github_session"
 end

data/spec/commenter/comment_sheet_spec.rb CHANGED Viewed

@@ -169,6 +169,31 @@ RSpec.describe Commenter::CommentSheet do
       expect(sheet.comments.length).to eq(1)
     end
   end
+end
+RSpec.describe Commenter::CommentSheet do
+  describe "#schema_name" do
+    it "selects the schema for each sheet version" do
+      expect(described_class.new(version: "osd").schema_name).to eq("iso_comment_osd.yaml")
+      expect(described_class.new(version: "2012-03").schema_name).to eq("iso_comment_2012-03.yaml")
+      expect(described_class.new({}).schema_name).to eq("iso_comment_2012-03.yaml")
+    end
+  end
+  describe "#to_yaml_document" do
+    it "prepends a header referencing the schema matching the version" do
+      document = described_class.new(version: "osd", comments: []).to_yaml_document
+      expect(document.lines.first).to eq("# yaml-language-server: $schema=schema/iso_comment_osd.yaml\n")
+      expect(YAML.safe_load(document)).to include("version" => "osd")
+    end
+    it "accepts a schema directory" do
+      document = described_class.new(comments: []).to_yaml_document("schemas")
+      expect(document.lines.first).to eq("# yaml-language-server: $schema=schemas/iso_comment_2012-03.yaml\n")
+    end
+  end
   describe "stage validation" do
     it "accepts valid stage values" do

data/spec/commenter/comment_type_spec.rb ADDED Viewed

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+require "spec_helper"
+RSpec.describe Commenter::CommentType do
+  describe ".code" do
+    it "maps full names and codes to codes" do
+      expect(described_class.code("Editorial")).to eq("ed")
+      expect(described_class.code("general")).to eq("ge")
+      expect(described_class.code("te")).to eq("te")
+    end
+    it "passes unrecognized values through" do
+      expect(described_class.code("custom")).to eq("custom")
+      expect(described_class.code("")).to eq("")
+      expect(described_class.code(nil)).to be_nil
+    end
+  end
+  describe ".full_name" do
+    it "expands codes to full names" do
+      expect(described_class.full_name("ge")).to eq("general")
+      expect(described_class.full_name("TE")).to eq("technical")
+      expect(described_class.full_name("editorial")).to eq("editorial")
+    end
+    it "passes unrecognized values through" do
+      expect(described_class.full_name("Custom")).to eq("Custom")
+      expect(described_class.full_name(nil)).to be_nil
+    end
+  end
+  describe ".display_name" do
+    it "capitalizes recognized types for display" do
+      expect(described_class.display_name("ge")).to eq("General")
+      expect(described_class.display_name("technical")).to eq("Technical")
+      expect(described_class.display_name("ed")).to eq("Editorial")
+    end
+    it "defaults unknown or missing types" do
+      expect(described_class.display_name("unknown")).to eq("unknown")
+      expect(described_class.display_name(nil)).to eq("Unknown")
+    end
+  end
+end

data/spec/commenter/github_integration_spec.rb CHANGED Viewed

@@ -135,49 +135,82 @@ RSpec.describe Commenter::GitHubIssueCreator do
   describe "template variable generation" do
     let(:creator) { described_class.new(config_file.path, title_template_file.path, body_template_file.path) }
-    let(:comment_sheet) { Commenter::CommentSheet.from_hash(yaml_data) }
-    let(:comment) { comment_sheet.comments.first }
-    it "generates correct template variables" do
-      variables = creator.send(:template_variables, comment, comment_sheet)
-      expect(variables["stage"]).to eq("DIS")
-      expect(variables["document"]).to eq("Test Document")
-      expect(variables["comment_id"]).to eq("US-001")
-      expect(variables["type"]).to eq("technical")
-      expect(variables["type_full_name"]).to eq("Technical")
-      expect(variables["clause"]).to eq("5.1")
-      expect(variables["element"]).to eq("Table 1")
-      expect(variables["comments"]).to eq("Test comment text")
-      expect(variables["brief_summary"]).to include("Clause 5.1")
-      expect(variables["has_proposed_change"]).to be true
+    let(:title_template_file) do
+      file = Tempfile.new(["title", ".liquid"])
+      file.write("{{ stage }}|{{ document }}|{{ project }}|{{ version }}|{{ comment_id }}|{{ type }}|" \
+                 "{{ type_full_name }}|{{ clause }}|{{ element }}|{{ line_number }}|" \
+                 "{{ has_observations }}|{{ has_proposed_change }}|{{ locality_summary }}|{{ unique_id }}")
+      file.close
+      file
+    end
+    it "exposes sheet, comment, and computed variables to templates" do
+      result = creator.create_issues_from_yaml(yaml_file.path, dry_run: true).first
+      variables = result[:title].split("|")
+      expect(variables).to eq(
+        [
+          "DIS", "Test Document", "Test Project", "2012-03", "US-001", "technical",
+          "Technical", "5.1", "Table 1", "", "false", "true", "Clause 5.1, Table 1", "[DIS] US-001"
+        ]
+      )
     end
   end
   describe "label determination" do
     let(:creator) { described_class.new(config_file.path, title_template_file.path, body_template_file.path) }
-    let(:comment_sheet) { Commenter::CommentSheet.from_hash(yaml_data) }
-    let(:comment) { comment_sheet.comments.first }
-    it "combines default, stage-specific, and comment type labels" do
-      labels = creator.send(:determine_labels, comment, comment_sheet)
+    it "combines default, stage-specific, and comment type labels without duplicates" do
+      result = creator.create_issues_from_yaml(yaml_file.path, dry_run: true).first
+      labels = result[:labels]
-      expect(labels).to include("comment-review") # default
-      expect(labels).to include("draft-international-standard") # stage-specific
-      expect(labels).to include("technical") # comment type (expanded)
-      expect(labels.uniq).to eq(labels) # no duplicates
+      expect(labels).to include("comment-review", "draft-international-standard", "technical")
+      expect(labels.uniq).to eq(labels)
     end
   end
+end
-  describe "comment type expansion" do
-    let(:creator) { described_class.new(config_file.path, title_template_file.path, body_template_file.path) }
+RSpec.describe Commenter::GitHubIssueRetriever do
+  let(:config_file) do
+    file = Tempfile.new(["config", ".yaml"])
+    file.write({ "github" => { "repository" => "test-org/test-repo", "token" => "test-token" } }.to_yaml)
+    file.close
+    file
+  end
+  let(:osd_yaml_file) do
+    file = Tempfile.new(["comments", ".yaml"])
+    file.write({
+      "version" => "osd",
+      "document" => "ISO/DIS 5843-6(en)",
+      "stage" => "DIS",
+      "comments" => [
+        {
+          "id" => "1",
+          "body" => "John Doe",
+          "locality" => { "clause" => "5.2.1" },
+          "type" => "editorial",
+          "comments" => "The values in Table 3 are inconsistent."
+        }
+      ]
+    }.to_yaml)
+    file.close
+    file
+  end
+  after do
+    config_file.unlink
+    osd_yaml_file.unlink
+  end
+  describe "#retrieve_observations_from_yaml" do
+    it "rewrites the YAML with the schema header matching its version" do
+      retriever = described_class.new(config_file.path)
+      retriever.retrieve_observations_from_yaml(osd_yaml_file.path)
-    it "expands comment type codes correctly" do
-      expect(creator.send(:expand_comment_type, "ge")).to eq("General")
-      expect(creator.send(:expand_comment_type, "te")).to eq("Technical")
-      expect(creator.send(:expand_comment_type, "ed")).to eq("Editorial")
-      expect(creator.send(:expand_comment_type, "unknown")).to eq("unknown")
-      expect(creator.send(:expand_comment_type, nil)).to eq("Unknown")
+      expect(osd_yaml_file.open.read.lines.first)
+        .to eq("# yaml-language-server: $schema=schema/iso_comment_osd.yaml\n")
     end
   end
 end