commenter 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31fa0d4ee056c833d9f2b329d994e5d6151e3430426e6d4f02dfca4c670d0c18
-  data.tar.gz: b459baef384850cfa84cfb22c48212771ee57fa0e11cbd803c1db33dfb44aae2
+  metadata.gz: 99f441f58237864e8ea0bc5c8682f04d2b15757ef1f28c45317f0181d6621fb4
+  data.tar.gz: 0a48406aa86377612a2722455dad5013457a10805da311c74c1fbfb82cccafd7
 SHA512:
-  metadata.gz: 989ff9a26a63210438385658edaee8a0d94a35c9d80916dfefe89575ce739f9efc75bdcc372d737a955de236fdefdeaaca0be65146363dea5833ac280cc97cdb
-  data.tar.gz: 5cdcb2b0ca204b6e419d009f45eeb6dfd4867c7eb51196a04ad2499c7c64e5cb5288b85522f031a7a823e9242c3e8edec771d7c2d1bb8f5f9b079df7e86cc54b
+  metadata.gz: 7c26ede9e2a9e4f5f060c0b8679cc9afcb36214a4c5b1a2f02c15a44cd6671d7b104954f4cf4b6df3df9f3806ab7c4d0418982a75371805c773c90064b82c8b9
+  data.tar.gz: 32dd2e31a603391bef5a112d0b10da62b7c29e249bcc76c3e5456aab939015ff51d21036108c3366743db11e01411a2bd285b7992f1a534009d5d4cc184e9d31

data/.rubocop.yml

@@ -8,3 +8,7 @@ Style/StringLiterals:
 
 Style/StringLiteralsInInterpolation:
   EnforcedStyle: double_quotes
+
+Metrics/ModuleLength:
+  Exclude:
+    - "spec/**/*"

data/CLAUDE.md

@@ -0,0 +1,58 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Commands
+
+```sh
+bin/setup                        # install dependencies
+bundle exec rake                 # default task: specs + RuboCop (this is what CI runs)
+bundle exec rspec                # all specs
+bundle exec rspec spec/commenter/comment_spec.rb          # single spec file
+bundle exec rspec spec/commenter/comment_spec.rb:42       # single example
+bundle exec rubocop              # lint (part of default rake task — must be clean)
+bundle exec rubocop -a           # autocorrect
+bundle exec exe/commenter        # run the CLI locally
+bin/console                      # interactive prompt
+```
+
+CI (metanorma/ci `generic-rake.yml`) runs `bundle exec rake` on Ruby 3.2/3.4/4.0 across macOS/Ubuntu/Windows. RuboCop offenses fail CI — run `bundle exec rake`, not just rspec, before pushing. `Gemfile.lock` is gitignored; dependencies resolve fresh in CI.
+
+Specs generate XLSX fixtures at runtime via `spec/support/xlsx_builder.rb` (plain rubyzip, no spreadsheet-writing dependency) plus row data in `spec/support/osd_fixtures.rb` — there are no binary fixture files in the repo.
+
+## What this gem does
+
+Converts ISO comment sheets to structured YAML and back, and syncs comments to GitHub issues:
+
+- **Import**: DOCX (ISO 2012-03 balloting template) or XLSX (ISO Online Standards Development exports) → YAML + schema file. Format auto-detected from extension.
+- **Fill**: YAML → filled DOCX comment sheet (`data/iso_comment_template_2012-03.docx`), optional status-based cell shading.
+- **GitHub round-trip**: `github-create` makes issues from YAML via Liquid templates; `github-retrieve` pulls `> **OBSERVATION:**` blockquotes from closed issues back into the YAML's observations field.
+
+Plain text only — formulas/images/complex formatting are unsupported (docx gem limitation).
+
+## Architecture
+
+Core flow: `Parser` → `CommentSheet` (metadata + `Comment` objects) → YAML / DOCX / GitHub issues.
+
+- `Commenter::Comment` / `Commenter::CommentSheet` — data model. `Comment` carries the common fields plus optional OSD-specific fields (`user_name`, `resolution_status`, `motivation`, etc.) and a `github` sub-hash tracking issue state. The sheet's `version` field (`"2012-03"` vs `"osd"`) selects the output schema.
+- `Commenter::Parser` — dispatches by format: `.docx` parsed inline with the `docx` gem; `.xlsx` delegated to `Parser::OsdXlsxParser` (uses `roo`).
+- `Parser::OsdXlsxParser` — auto-detects two OSD export variants by header row ("resolved" 17-col starting `Comment ID`; "unresolved" 15-col starting `User name`), extracts metadata (date/reference/stage/titles) from header rows 1–2, maps columns by header name into `Comment` attributes, and synthesizes `observations` from `resolution_status` + `motivation`.
+- `Commenter::Filler` — writes comments into the DOCX template table.
+- `Commenter::GitHubIssueCreator` (`lib/commenter/github_integration.rb`) — Octokit-based; duplicate detection searches for a unique ID rendered from a configurable Liquid `unique_id` template (stage-aware by default: `[DIS] GB-001`), so the same comment ID at different ballot stages creates separate issues.
+- `Commenter::Cli` (`lib/commenter/cli.rb`) — Thor CLI with subcommands `import`, `fill`, `github-create`, `github-retrieve`.
+
+### Schemas
+
+`schema/iso_comment_2012-03.yaml` and `schema/iso_comment_osd.yaml` are JSON-Schema-style files for IDE validation. `import` copies the matching schema (based on sheet version) into the schema dir and stamps the YAML header with a `yaml-language-server` reference. When the data model changes, update the corresponding schema file.
+
+### Templates
+
+`data/github_issue_title_template.liquid` and `data/github_issue_body_template.liquid` render issue titles/bodies; both are user-overridable via config. See README.adoc for the full variable list.
+
+## Conventions
+
+- Ruby style: double-quoted strings (enforced by RuboCop).
+- `data/` files (DOCX template, Liquid templates, sample config) are shipped with the gem and loaded relative to `__dir__` — do not treat them as disposable.
+- Testing GitHub integration: use `--dry-run` with `GITHUB_TOKEN=dummy_token` — exercises template rendering without API calls.
+- `sig/commenter.rbs` is a minimal stub; RBS coverage is not currently maintained.
+- Release: pushing a `v*` tag triggers `.github/workflows/release.yml`. Version lives in `lib/commenter/version.rb`.

data/README.adoc

@@ -7,14 +7,18 @@ image:https://img.shields.io/github/commits-since/metanorma/commenter/latest.svg
 
 == Purpose
 
-Commenter is a Ruby gem for working with ISO comment sheets in DOCX format.
+Commenter is a Ruby gem for working with ISO comment sheets in DOCX and XLSX
+format.
 
 It provides utilities for parsing, manipulating, and serializing ISO comment
-data, converting between DOCX and structured YAML with schema validation.
+data, converting between DOCX/XLSX and structured YAML with schema validation.
 
-The format is taken from:
+The supported input formats are:
 
 * "ISO/IEC/CEN/CENELEC electronic balloting commenting template/version 2012-03"
+  (DOCX)
+* ISO Online Standards Development (OSD) XLSX exports (resolved and unresolved
+  comment sheets)
 
 This gem only supports plain text comment extraction and filling. Only use this
 to handle plain text comments and resolutions.
@@ -49,19 +53,27 @@ $ gem install commenter
 
 == Usage
 
-=== Importing comments from DOCX
+=== Importing comments
 
-Convert an ISO comment sheet DOCX file to structured YAML:
+Convert an ISO comment sheet (DOCX or XLSX) to structured YAML.
+The format is auto-detected from the file extension.
 
 [source,shell]
 ----
+# From DOCX (ISO 2012-03 template or OSD DOCX export)
 $ commenter import "ISO 80000-2 review comments.docx" -o comments.yaml
+
+# From XLSX (ISO OSD resolved comment export)
+$ commenter import "91855-Comments-resolved.xlsx" -o comments.yaml
+
+# From XLSX (ISO OSD comments-only export)
+$ commenter import "ISO 5843-6-Comments.xlsx" -o comments.yaml
 ----
 
 This will create two files:
 
 `comments.yaml`:: The structured comment data
-`schema/iso_comment_2012-03.yaml`:: The YAML schema for validation
+`schema/iso_comment_2012-03.yaml` or `schema/iso_comment_osd.yaml`:: The YAML schema for validation (selected automatically based on input format)
 
 ==== Import options
 
@@ -75,6 +87,10 @@ Options:
 `-o, --output FILE`:: Output YAML file (default: comments.yaml)
 `-e, --exclude-observations`:: Skip the observations column
 `--schema-dir DIR`:: Directory for schema file (default: schema)
+`--format FORMAT`:: Force input format (`docx` or `xlsx`; auto-detected by default)
+`--sheet NAME`:: XLSX sheet name to parse (default: first sheet)
+`--resolved-only`:: XLSX: use resolved comments sheet only
+`--unresolved-only`:: XLSX: use unresolved comments sheet only
 
 ==== Metadata (extraction limitation)
 
@@ -530,12 +546,17 @@ The comment types are defined as follows:
 `te`:: Technical comment
 `ed`:: Editorial comment
 
+NOTE: In generated YAML, types are stored in expanded form (`general`,
+`technical`, `editorial`). Short codes are accepted as input anywhere and are
+expanded when the comment is loaded.
+
 === Workflow integration
 
 [source,mermaid]
 ----
 flowchart LR
-    A[ISO Comment Sheet DOCX] --> B[commenter import]
+    A1[ISO Comment Sheet DOCX] --> B[commenter import]
+    A2[ISO OSD XLSX export] --> B
     B --> C[YAML + Schema]
     C --> D[commenter github-create]
     D --> E[YAML + GitHub Info]
@@ -566,6 +587,8 @@ recognized and applied to the observations column:
 
 == Data model
 
+=== ISO 2012-03 template (DOCX)
+
 The comment structure follows this schema:
 
 [source,yaml]
@@ -593,14 +616,61 @@ comments:           # Array of comment objects
       updated_at: string      # ISO 8601 timestamp (optional)
 ----
 
+=== ISO OSD format (XLSX)
+
+The OSD format includes additional metadata and resolution fields extracted from
+the ISO Online Standards Development platform:
+
+[source,yaml]
+----
+version: "osd"           # OSD format identifier
+date: string             # Date from XLSX header (auto-extracted)
+document: string         # Document reference, e.g. "ISO/DIS 5843-6(en)"
+project: string          # Project name (auto-extracted)
+stage: string            # Stage: WD, CD, DIS, FDIS, PRF, PUB (auto-extracted)
+title_en: string         # Document title in English (auto-extracted)
+title_fr: string         # Document title in French (auto-extracted)
+comments:
+  - id: string           # Comment ID (numeric from OSD)
+    body: string         # User name / member body
+    locality:
+      clause: string     # Clause number
+      element: string    # Clause title
+    type: "ge" | "te" | "ed"
+    comments: string     # Comment text
+    proposed_change: string
+    observations: string # Built from resolution_status + motivation
+    user_name: string         # OSD user name
+    comment_type: string      # Subtype: Editorial, General, Technical
+    resolution_status: string # Accepted, Partially accepted, Rejected, etc.
+    resolution_date: string   # Date resolved
+    feedbacks: string         # Discussion replies
+    motivation: string        # Resolution justification
+    created_date: string      # Date comment was created
+    stage_code: string        # ISO stage code (e.g. "40.20")
+----
+
+The OSD XLSX format comes in two variants, both auto-detected:
+
+*Resolved*: 17 columns starting with `Comment ID`, includes resolution data
+(`Resolution status`, `Motivation`, `Resolution Date`, `Stage code`)
+
+*Unresolved/comments-only*: 15 columns starting with `User name`, includes
+`Comment type`, `Comment/Motivation`, `Replies`, `Comment number`
+
 
 == Schema validation
 
-Each exported YAML file includes a schema reference for IDE support:
+Each exported YAML file includes a schema reference for IDE support.
+The schema is selected automatically based on the input format:
 
 [source,yaml]
 ----
+# For ISO 2012-03 DOCX imports:
 # yaml-language-server: $schema=schema/iso_comment_2012-03.yaml
+
+# For ISO OSD XLSX imports:
+# yaml-language-server: $schema=schema/iso_comment_osd.yaml
 ----
 
 This enables:
@@ -695,7 +765,8 @@ The gem is organized into several key components:
 
 `Commenter::Comment`:: Represents individual comments with locality, type, and content
 `Commenter::CommentSheet`:: Container for multiple comments with metadata
-`Commenter::Parser`:: Handles DOCX parsing and YAML generation
+`Commenter::Parser`:: Handles DOCX and XLSX parsing with auto-detection
+`Commenter::Parser::OsdXlsxParser`:: Parses ISO OSD XLSX exports (resolved and unresolved variants)
 `Commenter::Filler`:: Fills DOCX templates with comment data
 `Commenter::GitHubIssueCreator`:: Creates GitHub issues from comments
 
@@ -703,7 +774,7 @@ The gem is organized into several key components:
 
 `Commenter::Cli`:: Thor-based command-line interface with subcommands:
 
-** `import` - Convert DOCX to YAML
+** `import` - Convert DOCX or XLSX to YAML
 ** `fill` - Fill DOCX template from YAML
 ** `github-create` - Create GitHub issues from comments
 ** `github-retrieve` - Retrieve observations from GitHub issues
@@ -714,7 +785,8 @@ The gem is organized into several key components:
 * `data/github_issue_title_template.liquid` - GitHub issue title template
 * `data/github_issue_body_template.liquid` - GitHub issue body template
 * `data/github_config_sample.yaml` - Sample GitHub configuration
-* `schema/iso_comment_2012-03.yaml` - YAML schema for validation
+* `schema/iso_comment_2012-03.yaml` - YAML schema for ISO 2012-03 validation
+* `schema/iso_comment_osd.yaml` - YAML schema for ISO OSD format validation
 
 === Debugging
 

data/commenter.gemspec

@@ -12,8 +12,8 @@ Gem::Specification.new do |spec|
   spec.authors = ["Ribose"]
   spec.email = ["open.source@ribose.com"]
 
-  spec.summary = "Library to work with ISO comment sheets in DOCX format."
-  spec.description = "Convert between ISO comment sheet DOCX and structured YAML with schema validation."
+  spec.summary = "Library to work with ISO comment sheets in DOCX and XLSX formats."
+  spec.description = "Convert between ISO comment sheets (DOCX and XLSX) and structured YAML with schema validation."
   spec.homepage = "https://github.com/metanorma/commenter"
   spec.license = "BSD-2-Clause"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.6.0")
@@ -35,5 +35,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dotenv", "~> 2.8"
   spec.add_dependency "liquid", "~> 5.0"
   spec.add_dependency "octokit", "~> 6.0"
+  spec.add_dependency "roo", ">= 2.10"
   spec.add_dependency "thor", "~> 1.0"
 end

data/lib/commenter/cli.rb

@@ -9,37 +9,43 @@ require "commenter/github_integration"
 
 module Commenter
   class Cli < Thor
-    desc "import INPUT.docx", "Convert DOCX comment sheet to YAML"
+    desc "import INPUT", "Convert comment sheet (DOCX or XLSX) to YAML"
     option :output, type: :string, aliases: :o, default: "comments.yaml", desc: "Output YAML file"
     option :exclude_observations, type: :boolean, aliases: :e, desc: "Exclude observations column"
     option :schema_dir, type: :string, default: "schema", desc: "Directory for schema file"
-    def import(input_docx)
+    option :format, type: :string, desc: "Force input format (docx or xlsx)"
+    option :sheet, type: :string, desc: "XLSX sheet name to parse (default: first sheet)"
+    option :resolved_only, type: :boolean, desc: "XLSX: use resolved comments sheet only"
+    option :unresolved_only, type: :boolean, desc: "XLSX: use unresolved comments sheet only"
+    def import(input_file)
       output_yaml = options[:output]
       schema_dir = options[:schema_dir]
 
       # Ensure schema directory exists
       FileUtils.mkdir_p(schema_dir) unless Dir.exist?(schema_dir)
 
-      # Parse the DOCX file
+      # Parse the input file
       parser = Parser.new
-      comment_sheet = parser.parse(input_docx, options)
+      comment_sheet = parser.parse(input_file, options)
+
+      # Determine which schema to use based on version
+      schema_name = comment_sheet.version == "osd" ? "iso_comment_osd.yaml" : "iso_comment_2012-03.yaml"
 
       # Write the YAML data file with schema reference
-      yaml_content = generate_yaml_with_header(comment_sheet.to_yaml_h, schema_dir)
+      yaml_content = generate_yaml_with_header(comment_sheet.to_yaml_h, schema_dir, schema_name)
       File.write(output_yaml, yaml_content)
 
       # Copy schema file to output directory
-      schema_source = File.join(__dir__, "../../schema/iso_comment_2012-03.yaml")
-      schema_target = File.join(schema_dir, "iso_comment_2012-03.yaml")
+      schema_source = File.join(__dir__, "../../schema/#{schema_name}")
+      schema_target = File.join(schema_dir, schema_name)
 
       # Only copy if source and target are different
-      unless File.expand_path(schema_source) == File.expand_path(schema_target)
-        FileUtils.cp(schema_source,
-                     schema_target)
-      end
+      FileUtils.cp(schema_source, schema_target) unless File.expand_path(schema_source) == File.expand_path(schema_target)
 
-      puts "Converted #{input_docx} to #{output_yaml}"
-      puts "Schema file created at #{schema_target}"
+      puts "Converted #{input_file} to #{output_yaml}"
+      puts "  Version: #{comment_sheet.version}"
+      puts "  Comments: #{comment_sheet.comments.length}"
+      puts "  Schema: #{schema_target}"
     end
 
     desc "fill INPUT.yaml", "Fill DOCX template from YAML comments"
@@ -209,8 +215,8 @@ module Commenter
 
     private
 
-    def generate_yaml_with_header(data, schema_dir)
-      schema_path = File.join(schema_dir, "iso_comment_2012-03.yaml")
+    def generate_yaml_with_header(data, schema_dir, schema_name = "iso_comment_2012-03.yaml")
+      schema_path = File.join(schema_dir, schema_name)
       header = "# yaml-language-server: $schema=#{schema_path}\n\n"
       header + data.to_yaml
     end

data/lib/commenter/comment.rb

@@ -2,7 +2,9 @@
 
 module Commenter
   class Comment
-    attr_accessor :id, :body, :locality, :type, :comments, :proposed_change, :observations, :github
+    attr_accessor :id, :body, :locality, :type, :comments, :proposed_change, :observations, :github,
+                  :user_name, :comment_type, :resolution_status, :resolution_date, :feedbacks,
+                  :motivation, :created_date, :stage_code
 
     def initialize(attributes = {})
       # Normalize input to symbols
@@ -16,8 +18,21 @@ module Commenter
       @proposed_change = attrs[:proposed_change]
       @observations = attrs[:observations]
       @github = symbolize_keys(attrs[:github] || {})
+
+      # OSD-specific fields
+      @user_name = attrs[:user_name]
+      @comment_type = attrs[:comment_type]
+      @resolution_status = attrs[:resolution_status]
+      @resolution_date = attrs[:resolution_date]
+      @feedbacks = attrs[:feedbacks]
+      @motivation = attrs[:motivation]
+      @created_date = attrs[:created_date]
+      @stage_code = attrs[:stage_code]
     end
 
+    # Expands short type codes to full names. One-way by design: already
+    # expanded values pass through unchanged so that reloading YAML output
+    # stays stable.
     def expand_comment_type(type)
       case type&.downcase
       when "ge" then "general"
@@ -117,6 +132,14 @@ module Commenter
         comments: @comments,
         proposed_change: @proposed_change,
         observations: @observations,
+        user_name: @user_name,
+        comment_type: @comment_type,
+        resolution_status: @resolution_status,
+        resolution_date: @resolution_date,
+        feedbacks: @feedbacks,
+        motivation: @motivation,
+        created_date: @created_date,
+        stage_code: @stage_code,
         github: @github.empty? ? nil : @github
       }.compact
     end

data/lib/commenter/comment_sheet.rb

@@ -4,7 +4,7 @@ require_relative "comment"
 
 module Commenter
   class CommentSheet
-    attr_accessor :version, :date, :document, :project, :stage, :comments
+    attr_accessor :version, :date, :document, :project, :stage, :comments, :title_en, :title_fr
 
     def initialize(attributes = {})
       # Normalize input to symbols
@@ -15,6 +15,8 @@ module Commenter
       @document = attrs[:document]
       @project = attrs[:project]
       @stage = attrs[:stage]
+      @title_en = attrs[:title_en]
+      @title_fr = attrs[:title_fr]
       @comments = (attrs[:comments] || []).map { |c| c.is_a?(Comment) ? c : Comment.from_hash(c) }
     end
 
@@ -29,12 +31,18 @@ module Commenter
         document: @document,
         project: @project,
         stage: @stage,
+        title_en: @title_en,
+        title_fr: @title_fr,
         comments: @comments.map(&:to_h)
       }
     end
 
     def to_yaml_h
-      stringify_keys(to_h.merge(comments: @comments.map(&:to_yaml_h)))
+      hash = to_h.merge(comments: @comments.map(&:to_yaml_h))
+      # Remove nil-valued keys for cleaner YAML output
+      hash.delete(:title_en) if hash[:title_en].nil?
+      hash.delete(:title_fr) if hash[:title_fr].nil?
+      stringify_keys(hash)
     end
 
     def self.from_hash(hash)