commenter 0.2.3 → 0.3.1

data/lib/commenter/parser/track_change_docx_parser.rb

@@ -0,0 +1,332 @@
+# frozen_string_literal: true
+
+require "zip"
+require "nokogiri"
+require_relative "../comment"
+require_relative "../comment_sheet"
+
+module Commenter
+  class Parser
+    # Extracts tracked changes from a redlined Word document into an ISO
+    # 2012-03 comment sheet.
+    #
+    # Captured track changes (w:ins, w:del, w:moveFrom, w:moveTo):
+    # - proposed_change renders the change itself, e.g. +Insert: "text"+
+    # - locality.clause is resolved from the nearest preceding heading
+    # - locality.element is resolved from caption/inline references
+    #   (Table N, Figure N, Formula (N), NOTE n) within the current clause
+    # - observations can be stamped via the observations or accept_all options
+    #
+    # Reviewer comment threads (word/comments.xml w:comment) are emitted
+    # after the track changes with -CNNN ids: the remark verbatim in
+    # comments, its instruction reworded as the proposed change, and empty
+    # observations for the owner to draft.
+    #
+    # Self-closing markers (paragraph-mark insertions inside rPr) carry no
+    # content and are skipped.
+    #
+    # word/document.xml is streamed with Nokogiri::XML::Reader because redline
+    # documents can exceed 100 MB.
+    class TrackChangeDocxParser
+      W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+
+      HEADING_STYLE = /\A(?:Heading[1-6]|h[2-5]annex\d*|ANNEX|BaseHeading)\z/i
+      KIND_COMMENT = { "ins" => "insertion", "del" => "deletion", "moveFrom" => "move (from)",
+                       "moveTo" => "move (to)" }.freeze
+      KIND_LABEL = { "ins" => "Insert", "del" => "Delete", "moveFrom" => "Move from",
+                     "moveTo" => "Move to" }.freeze
+
+      DEFAULT_BODY = "CS"
+      WHOLE_DOCUMENT = "_whole document"
+      ACCEPT_ALL = "Accepted. Tracked change accepted."
+
+      attr_reader :skipped_markers
+
+      def parse(path, options = {})
+        body = options[:body] || DEFAULT_BODY
+        changes = []
+        remarks = {}
+        anchors = {}
+        @skipped_markers = 0
+
+        Zip::File.open(path) do |zip|
+          entry = zip.glob("word/document.xml").first
+          raise Commenter::Error, "word/document.xml not found in #{path}" unless entry
+
+          remarks = read_remarks(zip)
+          changes = stream_changes(entry.get_input_stream, anchors)
+        end
+
+        CommentSheet.new(
+          version: "2012-03",
+          date: sheet_date(changes),
+          document: options[:document],
+          stage: options[:stage],
+          comments: build_change_comments(changes, body, options) +
+                    build_remark_comments(remarks, anchors, body, options)
+        )
+      end
+
+      # Turns a reviewer's remark phrased as a request ("Please remove this
+      # NOTE...") into the corresponding proposed change ("Remove this
+      # NOTE...").
+      def self.reword_remark(text)
+        text.to_s.strip.sub(/\APlease\s+/i, "").sub(/\A[a-z]/, &:upcase)
+      end
+
+      private
+
+      def read_remarks(zip)
+        entry = zip.glob("word/comments.xml").first
+        return {} unless entry
+
+        document = Nokogiri::XML(entry.get_input_stream)
+        document.xpath("//w:comment", "w" => W_NS).each_with_object({}) do |node, remarks|
+          remarks[node["w:id"]] = {
+            author: node["w:author"],
+            date: node["w:date"],
+            text: node.xpath(".//w:t", "w" => W_NS).map(&:text).join
+          }
+        end
+      end
+
+      def sheet_date(changes)
+        dates = changes.filter_map { |change| change[:date].to_s[/\A\d{4}-\d{2}-\d{2}/] }
+        dates.max
+      end
+
+      def build_change_comments(changes, body, options)
+        changes.each_with_index.map do |change, index|
+          kind = change[:kind]
+          Comment.new(
+            id: format("%<body>s-%03<index>d", body: body, index: index + 1),
+            body: body,
+            locality: { clause: change[:clause], line_number: nil, element: change[:element] },
+            type: options[:type] || "te",
+            comments: "Track change (#{KIND_COMMENT.fetch(kind, kind)})",
+            proposed_change: "#{KIND_LABEL.fetch(kind, kind)}: \"#{change[:text].strip}\"",
+            observations: observations_for(options)
+          )
+        end
+      end
+
+      def build_remark_comments(remarks, anchors, body, options)
+        remarks.keys.sort_by(&:to_i).each_with_index.map do |remark_id, index|
+          remark = remarks[remark_id]
+          anchor = anchors[remark_id] || {}
+          clause = anchor[:clause].to_s.empty? ? WHOLE_DOCUMENT : anchor[:clause]
+          Comment.new(
+            id: format("%<body>s-C%03<index>d", body: body, index: index + 1),
+            body: body,
+            locality: { clause: clause, line_number: nil, element: anchor[:element] },
+            type: options[:remark_type] || "ed",
+            comments: remark[:text],
+            proposed_change: self.class.reword_remark(remark[:text]),
+            observations: nil
+          )
+        end
+      end
+
+      def observations_for(options)
+        return nil if options[:exclude_observations]
+
+        options[:observations] || (options[:accept_all] ? ACCEPT_ALL : nil)
+      end
+
+      def stream_changes(io, anchors = {})
+        changes = []
+        change_stack = []
+        paragraph = ParagraphState.new
+
+        Nokogiri::XML::Reader(io).each do |reader|
+          case reader.node_type
+          when Nokogiri::XML::Reader::TYPE_ELEMENT
+            handle_start(reader, change_stack, paragraph, anchors)
+          when Nokogiri::XML::Reader::TYPE_TEXT, Nokogiri::XML::Reader::TYPE_SIGNIFICANT_WHITESPACE
+            handle_text(reader, change_stack, paragraph)
+          when Nokogiri::XML::Reader::TYPE_END_ELEMENT
+            handle_end(reader, change_stack, paragraph, changes)
+          end
+        end
+
+        changes
+      end
+
+      def handle_start(reader, change_stack, paragraph, anchors = {})
+        case reader.local_name
+        when "p"
+          paragraph.begin_paragraph
+        when "pStyle"
+          paragraph.style = reader.attribute("w:val") || reader.attribute("val")
+        when "ins", "del", "moveFrom", "moveTo"
+          start_change(reader, change_stack, paragraph)
+        when "commentRangeStart"
+          anchor_remark(reader, paragraph, anchors)
+        end
+      end
+
+      def anchor_remark(reader, paragraph, anchors)
+        id = reader.attribute("w:id")
+        anchors[id] = {
+          clause: paragraph.pending_clause || paragraph.clause,
+          element: paragraph.pending_element || paragraph.element
+        }
+      end
+
+      def start_change(reader, change_stack, paragraph)
+        if reader.empty_element?
+          @skipped_markers += 1
+          return
+        end
+
+        change_stack.push(
+          kind: reader.local_name,
+          id: reader.attribute("w:id"),
+          author: reader.attribute("w:author"),
+          date: reader.attribute("w:date"),
+          text: +"",
+          clause: paragraph.clause,
+          element: paragraph.element
+        )
+      end
+
+      def handle_text(reader, change_stack, paragraph)
+        if (current = change_stack.last)
+          current[:text] << reader.value
+        else
+          paragraph.append_text(reader.value)
+        end
+      end
+
+      def handle_end(reader, change_stack, paragraph, changes)
+        case reader.local_name
+        when "p"
+          paragraph.commit_paragraph
+        when "ins", "del", "moveFrom", "moveTo"
+          change = change_stack.pop
+          changes << finalize_change(change, paragraph) if change
+        end
+      end
+
+      def finalize_change(change, paragraph)
+        # A change inside a heading or caption paragraph belongs to that
+        # paragraph, which is not committed yet at this point.
+        change[:clause] = paragraph.pending_clause || change[:clause]
+        change[:element] = paragraph.pending_element || change[:element]
+        change[:clause] = WHOLE_DOCUMENT if change[:clause].to_s.empty?
+        change
+      end
+
+      # Tracks the current paragraph's style and text to resolve the locality
+      # context of a change:
+      #
+      # - clause: the clause number of the most recent heading paragraph; an
+      #   unnumbered sub-heading inherits its parent heading's clause
+      # - element: the nearest Table/Figure/Formula/NOTE reference, scoped to
+      #   the current clause (caption paragraphs and inline mentions both
+      #   count)
+      class ParagraphState
+        NUMBERED_CLAUSE = /\A(?:\d+(?:\.\d+)*|Annex\s+[A-Z](?:\.\d+)*|Bibliography|Foreword|Introduction)\z/i
+        ELEMENT_START = /\A\s*(NOTE\s+\d+|NOTE(?=\s*[—:-])|Table\s+(?:[A-Z]\.)?\d+(?:\.\d+)*|Figure\s+(?:[A-Z]\.)?\d+(?:\.\d+)*)\b/i
+        FORMULA = /\b(Formula\s*\(\d+(?:\.\d+)*\))/
+        ELEMENT_ANY = /\b(Table\s+(?:[A-Z]\.)?\d+(?:\.\d+)*|Figure\s+(?:[A-Z]\.)?\d+(?:\.\d+)*|NOTE\s+\d+)\b/i
+
+        attr_writer :style
+        attr_reader :clause, :element
+
+        def initialize
+          @style = nil
+          @text = +""
+          @clause = ""
+          @element = nil
+          @heading_stack = []
+        end
+
+        def begin_paragraph
+          @style = nil
+          @text = +""
+        end
+
+        def append_text(value)
+          @text << value
+        end
+
+        def pending_clause
+          return unless heading_paragraph?
+
+          clause = self.class.clause_for(@text)
+          clause unless clause.empty?
+        end
+
+        def pending_element
+          self.class.element_for(@text)
+        end
+
+        def commit_paragraph
+          if heading_paragraph?
+            commit_heading
+          else
+            @element = self.class.element_for(@text) || @element
+          end
+        end
+
+        def heading_paragraph?
+          @style&.match?(HEADING_STYLE)
+        end
+
+        def self.numbered_clause?(clause)
+          clause.to_s.match?(NUMBERED_CLAUSE)
+        end
+
+        # Extracts the clause identifier from heading text: the leading number
+        # ("4.2.1 Thermodynamic ..." -> "4.2.1"), an annex reference, or a
+        # well-known unnumbered section. Runs may be concatenated without
+        # spaces ("4.2.1Thermodynamic"), so the number is matched with a
+        # lookahead.
+        def self.clause_for(text)
+          s = text.to_s.strip
+          return "" if s.empty?
+
+          return Regexp.last_match(1) if s =~ /\A\s*(Annex\s+[A-Z](?:\.\d+)*)\b/i
+          return Regexp.last_match(1) if s =~ /\A\s*(Bibliography|Foreword|Introduction)\b/i
+          return Regexp.last_match(1) if s =~ /\A\s*((?:\d+\.)*\d+)(?=[A-Z\s])/
+          return Regexp.last_match(1) if s =~ /\A\s*((?:\d+\.)*\d+)\b/
+
+          s
+        end
+
+        # Resolves the element reference of a paragraph: a caption opening the
+        # paragraph ("Table 3 — ...", "NOTE 2 ...", "Formula (9):") wins over an
+        # inline mention ("the values in Table 5 shall ...").
+        def self.element_for(text)
+          s = text.to_s
+          match = s.match(ELEMENT_START) || s.match(FORMULA) || s.match(ELEMENT_ANY)
+          match[1].squeeze(" ").strip if match
+        end
+
+        private
+
+        def commit_heading
+          level = heading_level
+          clause = self.class.clause_for(@text)
+          clause = inherited_clause(level) unless self.class.numbered_clause?(clause)
+          @heading_stack.pop while @heading_stack.last && @heading_stack.last[0] >= level
+          @heading_stack << [level, clause]
+          @clause = clause unless clause.empty?
+          # Element references belong to the clause they appear in.
+          @element = nil
+        end
+
+        def inherited_clause(level)
+          parent = @heading_stack.reverse.find { |(lvl, _)| lvl < level }
+          parent ? parent[1] : ""
+        end
+
+        def heading_level
+          m = @style.match(/\A(?:Heading|h)(\d+)/i)
+          m ? m[1].to_i : 1
+        end
+      end
+    end
+  end
+end

data/lib/commenter/parser.rb

@@ -1,12 +1,51 @@
 # frozen_string_literal: true
 
 require "docx"
+require "pathname"
 require_relative "comment_sheet"
 require_relative "comment"
+require_relative "parser/osd_xlsx_parser"
+require_relative "parser/track_change_docx_parser"
 
 module Commenter
   class Parser
-    def parse(docx_path, options = {})
+    def parse(input_path, options = {})
+      format = detect_format(input_path, options)
+
+      case format
+      when :docx
+        parse_docx(input_path, options)
+      when :xlsx
+        parse_xlsx(input_path, options)
+      when :redline
+        parse_redline(input_path, options)
+      else
+        raise "Unsupported file format: #{input_path}. Supported formats: .docx, .xlsx"
+      end
+    end
+
+    private
+
+    def detect_format(path, options = {})
+      return options[:format].to_sym if options[:format]
+
+      ext = File.extname(path).downcase
+      case ext
+      when ".docx" then :docx
+      when ".xlsx", ".xls" then :xlsx
+      else :unknown
+      end
+    end
+
+    def parse_xlsx(xlsx_path, options)
+      OsdXlsxParser.new.parse(xlsx_path, options)
+    end
+
+    def parse_redline(docx_path, options)
+      TrackChangeDocxParser.new.parse(docx_path, options)
+    end
+
+    def parse_docx(docx_path, options)
       doc = Docx::Document.open(docx_path)
 
       # Extract metadata from the first table
@@ -17,41 +56,13 @@ module Commenter
       raise "No comments table found in document" unless comments_table
       raise "Comments table appears to be empty" if comments_table.row_count < 2
 
-      comments = []
-
       # Process all rows - don't skip any rows, respect all content
-      (0..comments_table.row_count - 1).each do |i|
-        row = comments_table.rows[i]
-        cells = row.cells.map { |c| c.text.strip }
-
-        # Skip only completely empty rows
+      comments = (0..comments_table.row_count - 1).map do |row_index|
+        cells = comments_table.rows[row_index].cells.map { |cell| cell.text.strip }
         next if cells.all?(&:empty?)
 
-        # Extract body from ID (e.g., "DE-001" -> "DE")
-        id = cells[0] || ""
-        body = id.include?("-") ? id.split("-").first : id
-
-        # Create comment with symbol keys, respecting all input data
-        comment_attrs = {
-          id: id,
-          body: body,
-          locality: {
-            line_number: cells[1] && cells[1].empty? ? nil : cells[1],
-            clause: cells[2] && cells[2].empty? ? nil : cells[2],
-            element: cells[3] && cells[3].empty? ? nil : cells[3]
-          },
-          type: cells[4] || "",
-          comments: cells[5] || "",
-          proposed_change: cells[6] || ""
-        }
-
-        # Handle observations column
-        unless options[:exclude_observations]
-          comment_attrs[:observations] = cells[7] && cells[7].empty? ? nil : cells[7]
-        end
-
-        comments << Comment.new(comment_attrs)
-      end
+        osd_docx_row?(cells) ? build_osd_docx_comment(cells, options) : build_classic_docx_comment(cells, options)
+      end.compact
 
       # Create comment sheet
       CommentSheet.new(
@@ -63,7 +74,55 @@ module Commenter
       )
     end
 
-    private
+    # The DOCX from ISO OSD has 9 columns:
+    # 0: User name, 1: (empty/line), 2: Clause nb, 3: Clause Title,
+    # 4: Type, 5: Comment, 6: (empty/proposal), 7: Observations, 8: Comment number
+    # Detected by checking if last column looks like a numeric ID.
+    def osd_docx_row?(cells)
+      cells.length >= 9 && cells[8].to_s.match?(/^\d+$/)
+    end
+
+    def build_osd_docx_comment(cells, options)
+      attrs = {
+        id: cells[8],
+        body: cells[0].to_s.strip,
+        locality: {
+          clause: presence(cells[2]),
+          element: presence(cells[3])
+        },
+        type: CommentType.code(cells[4]),
+        comments: cells[5] || "",
+        proposed_change: presence(cells[6]),
+        user_name: cells[0].to_s.strip
+      }
+      attrs[:observations] = presence(cells[7]) unless options[:exclude_observations]
+      Comment.new(attrs)
+    end
+
+    # Classic ISO comment template format (8 columns):
+    # 0: ID, 1: line_number, 2: clause, 3: element, 4: type, 5: comments,
+    # 6: proposed_change, 7: observations
+    def build_classic_docx_comment(cells, options)
+      id = cells[0] || ""
+      attrs = {
+        id: id,
+        body: id.include?("-") ? id.split("-").first : id,
+        locality: {
+          line_number: presence(cells[1]),
+          clause: presence(cells[2]),
+          element: presence(cells[3])
+        },
+        type: cells[4] || "",
+        comments: cells[5] || "",
+        proposed_change: cells[6] || ""
+      }
+      attrs[:observations] = presence(cells[7]) unless options[:exclude_observations]
+      Comment.new(attrs)
+    end
+
+    def presence(value)
+      value && !value.empty? ? value : nil
+    end
 
     def extract_metadata(doc)
       metadata = { date: nil, document: nil, project: nil }
@@ -96,9 +155,6 @@ module Commenter
       project_match = all_text.match(/Project:\s*([^\n\r]+)/)
       metadata[:project] = project_match[1]&.strip if project_match
 
-      # If no metadata found, try to extract from filename or other sources
-      # This is a fallback - in practice, users might need to provide metadata manually
-
       metadata
     end
   end

data/lib/commenter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Commenter
-  VERSION = "0.2.3"
+  VERSION = "0.3.1"
 end

data/lib/commenter.rb

@@ -7,6 +7,11 @@ require_relative "commenter/parser"
 require_relative "commenter/filler"
 require_relative "commenter/github_integration"
 
+# Commenter converts ISO comment sheets (DOCX/XLSX) to structured YAML and
+# syncs comments to GitHub issues.
 module Commenter
   class Error < StandardError; end
+
+  autoload :CommentType, "commenter/comment_type"
+  autoload :GitHubSession, "commenter/github_session"
 end

data/schema/iso_comment_2012-03.yaml

@@ -68,8 +68,8 @@ properties:
           required: ["clause"]
         type:
           type: string
-          enum: ["ge", "te", "ed"]
-          description: "Type of comment: ge=general, te=technical, ed=editorial"
+          enum: ["ge", "te", "ed", "general", "technical", "editorial"]
+          description: "Type of comment: ge=general, te=technical, ed=editorial. Codes are expanded to full names in output"
         comments:
           type: string
           description: "The actual comment text"

data/schema/iso_comment_osd.yaml

@@ -0,0 +1,112 @@
+# ISO Comment Schema for ISO Online Standards Development (OSD) format
+# Supports both resolved and unresolved XLSX exports from ISO OSD
+$schema: http://json-schema.org/draft-07/schema#
+title: ISO Comment OSD
+description: Schema for ISO comments exported from ISO Online Standards Development (OSD) platform
+type: object
+properties:
+  version:
+    type: string
+    const: "osd"
+    description: Version identifier for OSD format
+  date:
+    type: ["string", "null"]
+    description: Date from the XLSX header
+    format: date
+  document:
+    type: ["string", "null"]
+    description: Document reference (e.g. ISO/DIS 5843-6(en))
+  project:
+    type: ["string", "null"]
+    description: Project name
+  title_en:
+    type: ["string", "null"]
+    description: Document title in English
+  title_fr:
+    type: ["string", "null"]
+    description: Document title in French
+  stage:
+    type: ["string", "null"]
+    description: Approval stage (WD/CD/DIS/FDIS/PRF/PUB)
+    enum: [null, "WD", "CD", "DIS", "FDIS", "PRF", "PUB"]
+  comments:
+    type: array
+    description: Array of comment entries from the ISO OSD export
+    items:
+      type: object
+      properties:
+        id:
+          type: ["string", "number"]
+          description: "Comment ID (numeric from OSD)"
+        body:
+          type: ["string", "null"]
+          description: "Member body or user name"
+        locality:
+          type: object
+          description: "Location information for the comment"
+          properties:
+            line_number:
+              type: ["string", "null"]
+            clause:
+              type: ["string", "null"]
+            element:
+              type: ["string", "null"]
+              description: "Clause Title from OSD"
+        type:
+          type: ["string", "null"]
+          description: "Type of comment; codes (ge/te/ed) are expanded to general/technical/editorial in output"
+          enum: [null, "ge", "te", "ed", "general", "technical", "editorial"]
+        comments:
+          type: ["string", "null"]
+          description: "The comment text"
+        proposed_change:
+          type: ["string", "null"]
+          description: "Proposed change text"
+        observations:
+          type: ["string", "null"]
+          description: "Resolution status or observations"
+        user_name:
+          type: ["string", "null"]
+          description: "User name from OSD"
+        comment_type:
+          type: ["string", "null"]
+          description: "Comment type/subtype from OSD (e.g. Editorial, General, Technical)"
+        resolution_status:
+          type: ["string", "null"]
+          description: "Resolution status (Accepted, Partially accepted, Rejected, etc.)"
+        resolution_date:
+          type: ["string", "null"]
+          description: "Date when the comment was resolved"
+        feedbacks:
+          type: ["string", "null"]
+          description: "Feedback/replies from the discussion"
+        motivation:
+          type: ["string", "null"]
+          description: "Motivation/justification for resolution"
+        created_date:
+          type: ["string", "null"]
+          description: "Date when the comment was created"
+        stage_code:
+          type: ["string", "null"]
+          description: "ISO stage code (e.g. 40.20)"
+        github:
+          type: ["object", "null"]
+          description: "GitHub integration information"
+          properties:
+            issue_number:
+              type: integer
+            issue_url:
+              type: string
+              format: uri
+            status:
+              type: string
+              enum: ["open", "closed"]
+            created_at:
+              type: string
+              format: date-time
+            updated_at:
+              type: string
+              format: date-time
+          required: ["issue_number", "issue_url", "status"]
+      required: ["id", "locality", "comments"]
+required: ["version", "comments"]

data/spec/commenter/cli_spec.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "commenter/cli"
+require_relative "../support/osd_fixtures"
+require_relative "../support/xlsx_builder"
+
+RSpec.describe Commenter::Cli do
+  include OsdFixtures
+
+  describe "#import" do
+    it "writes OSD YAML and copies the OSD schema for XLSX input" do
+      Dir.mktmpdir do |dir|
+        xlsx = File.join(dir, "comments.xlsx")
+        rows = header_block(OsdFixtures::RESOLVED_HEADERS) + [
+          [1, "John Doe", "5.2.1", "Requirements", "Comment", "Editorial",
+           "The values in Table 3 are inconsistent.", nil, "Correct the values.", nil, nil, nil,
+           "2026-04-01", "Accepted", nil, "2026-04-10", "40.20"]
+        ]
+        XlsxBuilder.write(xlsx, [["Comments (1)", rows]])
+        output = File.join(dir, "out", "comments.yaml")
+        schema_dir = File.join(dir, "out", "schema")
+
+        described_class.start(["import", xlsx, "--output", output, "--schema-dir", schema_dir])
+
+        data = YAML.safe_load_file(output)
+        expect(data["version"]).to eq("osd")
+        expect(data["comments"].length).to eq(1)
+        expect(data["comments"].first["id"]).to eq("1")
+        expect(File.read(output)).to start_with("# yaml-language-server: $schema=")
+        expect(File).to exist(File.join(schema_dir, "iso_comment_osd.yaml"))
+      end
+    end
+  end
+end