ietf-data-importer 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0df7ab18290351d1560ddc5e0f0e1e5d976028045f22b6f4b87b89f25f747ffc
-  data.tar.gz: c814b6d33bced977dd20c9c0c3f0f8553007a095eb53116f6c740238eb29a870
+  metadata.gz: 0ec1ce1c3dff96c474c842a8e47047b251e1cc99586485c69dc38520b6c2c617
+  data.tar.gz: c746556a80d5b8575619df38f97d40a9dd6b713e53dbfb8ea5ecfd9b02dbea0e
 SHA512:
-  metadata.gz: 9b9d8d45a4d9997c0e8f3118c61140f8986bdb169d2379d42f7aee0ee8ef25f6b957db4f66531825a2aac415f940eb1f66b4f4d85c1d89b70fc66045b91e1018
-  data.tar.gz: aa884d901083552f00b086908bbfcc269a5b71d971d493d4333291b9ba02c774441dfb1e6c9b54eff4689ac0656624969e7fdb98f5b1716e5c8275635cea2342
+  metadata.gz: acfb7dc40db21492a495848becfd74bfe6cce844fdc3ea47531e16bbdb375ae1450fcfe0ff8bc5b347b1dbc91d7636e0377a877c4f89ec16192bca21f7bf8b15
+  data.tar.gz: 81586059d68dbc27f3a6d8a76dbe2cc19c4a64eb062cbcd2e333c44a0c9dbe7922ae1b382a9bf07ef8eda697bb27a1f4523224a0fe01439e7b7b4dfb81bf583a

data/.github/workflows/check_update.yml

@@ -10,7 +10,7 @@ jobs:
     name: Check workgroups update
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - run: |
           git config user.name github-actions
@@ -18,19 +18,19 @@ jobs:
 
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '2.6'
+          ruby-version: '3.0'
           bundler-cache: true
 
-      - id: update_json
+      - id: update
         run: |
-          bundle exec bin/wg2json.rb > lib/metanorma/ietf/data/workgroups.json
+          bundle exec exe/ietf-data-importer fetch lib/ietf/data/importer/groups.yaml
           git diff-index --quiet HEAD -- && has_changes="false" || has_changes="true"
-          echo "::set-output name=has_changes::${has_changes}"
+          echo "has_changes=${has_changes}" >> "$GITHUB_OUTPUT"
 
-      - if: ${{ !steps.update_json.outputs.has_changes == 'true' }}
+      - if: steps.update.outputs.has_changes == 'true'
         run: gem bump --version patch --tag --push
 
-      - if: ${{ !steps.update_json.outputs.has_changes == 'true' }}
+      - if: steps.update.outputs.has_changes == 'true'
         name: publish to rubygems.org
         env:
           RUBYGEMS_API_KEY: ${{secrets.METANORMA_CI_RUBYGEMS_API_KEY}}

data/.gitignore

@@ -1,2 +1,6 @@
 /.rubocop-https*
 /Gemfile.lock
+/.rspec_status
+/coverage/
+*.gem
+.DS_Store

data/.rubocop.yml

@@ -1,10 +1,17 @@
 # Auto-generated by Cimas: Do not edit it manually!
 # See https://github.com/metanorma/cimas
 inherit_from:
+  - .rubocop_todo.yml
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 
 # local repo-specific modifications
 # ...
 
+plugins:
+  - rubocop-performance
+
 AllCops:
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 3.0
+
+Metrics/MethodLength:
+  Max: 27

data/.rubocop_todo.yml

@@ -0,0 +1,49 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2026-05-12 10:00:04 UTC using RuboCop version 1.86.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 14
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings.
+# URISchemes: http, https
+Layout/LineLength:
+  Exclude:
+    - 'lib/ietf/data/importer/scrapers/ietf_scraper.rb'
+    - 'lib/ietf/data/importer/scrapers/irtf_scraper.rb'
+    - 'spec/ietf/data/importer/cli_spec.rb'
+    - 'spec/ietf/data/importer/group_collection_spec.rb'
+
+# Offense count: 6
+# Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
+Metrics/AbcSize:
+  Exclude:
+    - 'lib/ietf/data/importer/scrapers/ietf_scraper.rb'
+    - 'lib/ietf/data/importer/scrapers/irtf_scraper.rb'
+
+# Offense count: 4
+# Configuration parameters: AllowedMethods, AllowedPatterns, Max.
+Metrics/CyclomaticComplexity:
+  Exclude:
+    - 'lib/ietf/data/importer/scrapers/ietf_scraper.rb'
+    - 'lib/ietf/data/importer/scrapers/irtf_scraper.rb'
+
+# Offense count: 10
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
+Metrics/MethodLength:
+  Max: 27
+
+# Offense count: 4
+# Configuration parameters: AllowedMethods, AllowedPatterns, Max.
+Metrics/PerceivedComplexity:
+  Exclude:
+    - 'lib/ietf/data/importer/scrapers/ietf_scraper.rb'
+    - 'lib/ietf/data/importer/scrapers/irtf_scraper.rb'
+
+# Offense count: 1
+Security/Open:
+  Exclude:
+    - 'lib/ietf/data/importer/scrapers/base_scraper.rb'

data/CLAUDE.md

@@ -0,0 +1,73 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+`ietf-data-importer` is a Ruby gem providing offline access to IETF working group and IRTF research group metadata. It ships bundled YAML data and includes web scrapers to refresh it from `datatracker.ietf.org` and `irtf.org`.
+
+## Commands
+
+```sh
+bundle install          # install dependencies
+bundle exec rake        # run tests (default task = rspec)
+bundle exec rspec       # run full test suite
+bundle exec rspec spec/ietf/data/importer_spec.rb:45  # run single test by line
+bundle exec rubocop     # lint
+exe/ietf-data-importer fetch output.yaml             # scrape & write YAML
+exe/ietf-data-importer fetch output.json --format=json
+exe/ietf-data-importer integrate groups.yaml          # embed YAML into gem
+```
+
+## Architecture
+
+Namespace: `Ietf::Data::Importer`
+
+```
+lib/ietf/data/importer.rb              # Entry point — autoload, thin facade delegating to GroupCollection
+lib/ietf/data/importer/
+  version.rb                           # VERSION constant
+  group.rb                             # Group model (Lutaml::Model) with predicate methods
+  group_collection.rb                  # Rich collection model (Enumerable, query methods, merge, from_file, save)
+  groups.yaml                          # Bundled group data (shipped in gem)
+  cli.rb                               # Thor CLI (fetch / integrate commands)
+  scrapers.rb                          # Scrapers.fetch_all / fetch_ietf / fetch_irtf → GroupCollections
+  scrapers/
+    base_scraper.rb                    # Abstract: fetch_html, log, build_group, build_collection
+    ietf_scraper.rb                    # Scrapes datatracker.ietf.org → GroupCollection
+    irtf_scraper.rb                    # Scrapes irtf.org → GroupCollection
+```
+
+**Layer separation:**
+- **Models** (`Group`, `GroupCollection`) — all data and query logic. GroupCollection includes Enumerable and supports chainable filters returning new GroupCollections (`collection.active.by_type("wg")`).
+- **Facade** (`Importer` module) — loads bundled `groups.yaml` via `GroupCollection.from_file`, delegates query methods to the collection for backward-compatible API. Uses `autoload` (following sts-ruby pattern).
+- **Scrapers** — each returns a `GroupCollection`. `Scrapers.fetch_all` uses `merge` to combine results. `BaseScraper` provides `build_group`/`build_collection` template methods.
+- **CLI** (`Cli`) — thin Thor wrapper calling scraper and collection methods.
+
+**Key design decisions:**
+- Models use `lutaml-model` for serialization — attribute declarations + key_value mapping blocks, same as sts-ruby
+- GroupCollection filter methods (`by_organization`, `by_type`, `by_area`, `active`, `concluded`) return new GroupCollections, enabling chaining (open/closed principle)
+- All facade query methods return GroupCollection; only `groups` returns Array
+- `Group` has predicate methods (`active?`, `ietf?`, `working_group?`) — business logic on the model, not in collection filters
+- `GroupCollection.merge(other)` combines collections immutably
+- `GroupCollection.from_file(path)` / `#save(path, format:)` handle persistence
+- Entry point (`exe/ietf-data-importer`) requires the main importer file, triggering autoload — no direct requires in cli.rb
+- Tests inject data by stubbing `Importer.collection` — no private state access or `class_variable_set`
+- Scrapers are resilient to site layout changes: multiple CSS selectors tried as fallbacks
+- Scraper methods return values rather than mutating parameters
+
+## Conventions
+
+- Ruby 3.0+ required
+- Frozen string literals everywhere
+- Lutaml::Model for all data models (no raw Hash manipulation in the public API)
+- Test fixtures in `spec/fixtures/`; specs in `spec/ietf/data/importer/` (group_spec, group_collection_spec, cli_spec, importer_spec)
+- Tests stub `Importer.collection` scoped to `:query_tests` context — no global stubs, no private state access
+- Rubocop config inherits from riboseinc/oss-guides (remote URL in `.rubocop.yml`)
+
+## Reference Project
+
+The `sts-ruby` project at `../sts-ruby/` demonstrates the canonical patterns for Lutaml::Model-based gems in this org:
+- `autoload` for lazy-loading modules/classes
+- Each model is a single class in its own file, inheriting `Lutaml::Model::Serializable`
+- One file per class; no `private` send or `respond_to?` — rely on typed interfaces

data/Gemfile

@@ -5,7 +5,6 @@ source "https://rubygems.org"
 gemspec
 
 gem "rake"
-
 gem "rspec"
-
 gem "rubocop"
+gem "rubocop-performance"

data/README.adoc

@@ -7,7 +7,8 @@ image:https://img.shields.io/github/commits-since/metanorma/ietf-data-importer/l
 
 == Purpose
 
-IETF Data Importer is a Ruby gem providing access to information about IETF working groups and IRTF research groups.
+IETF Data Importer is a Ruby gem providing access to information about IETF
+working groups and IRTF research groups.
 
 It includes:
 
@@ -20,29 +21,6 @@ This gem exists because the official sources often change their layout or may be
 * https://datatracker.ietf.org/group/ (formerly tools.ietf.org/wg)
 * https://irtf.org/groups
 
-== Migration from metanorma-ietf-data
-
-This gem is the renamed and restructured version of `metanorma-ietf-data`. The namespace and file structure have been changed to match other Metanorma data importer gems.
-
-To migrate from metanorma-ietf-data:
-
-. Replace the following:
-+
-[source,diff]
-----
-- require "metanorma/ietf/data"
-- groups = Metanorma::Ietf::Data.groups
-+ require "ietf/data/importer"
-+ groups = Ietf::Data::Importer.groups
-----
-
-. Update your Gemfile:
-+
-[source,diff]
-----
-- gem 'metanorma-ietf-data'
-+ gem 'ietf-data-importer'
-----
 
 == Installation
 
@@ -211,6 +189,36 @@ groups:
 ----
 ====
 
+== Migration from metanorma-ietf-data
+
+The versions 0.1.0 and 0.2.0 of this gem were published under the name
+`metanorma-ietf-data`.
+
+The gem was rewritten and republished as `ietf-data-importer` to better reflect
+its purpose at version 0.3.0. The namespace and file structure have been changed
+to match other Metanorma data importer gems.
+
+To migrate from metanorma-ietf-data:
+
+. Replace the following:
++
+[source,diff]
+----
+- require "metanorma/ietf/data"
+- groups = Metanorma::Ietf::Data.groups
++ require "ietf/data/importer"
++ groups = Ietf::Data::Importer.groups
+----
+
+. Update your Gemfile:
++
+[source,diff]
+----
+- gem 'metanorma-ietf-data'
++ gem 'ietf-data-importer'
+----
+
+
 == Copyright
 
 This gem is developed, maintained and funded by https://www.ribose.com[Ribose Inc.]

data/exe/ietf-data-importer

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require_relative "../lib/ietf/data/importer/cli"
+require_relative "../lib/ietf/data/importer"
 
 Ietf::Data::Importer::Cli.start(ARGV)

data/ietf-data-importer.gemspec

@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
   spec.license       = "BSD-2-Clause"
   spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0")
 
-  spec.add_dependency "lutaml-model", "~> 0.7"
+  spec.add_dependency "lutaml-model", "~> 0.8"
+  spec.add_dependency "nokogiri", "~> 1.19"
   spec.add_dependency "thor", "~> 1.0"
-  spec.add_dependency "nokogiri", "~> 1.18"
   spec.add_dependency "yaml"
 
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
@@ -34,4 +34,5 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
+  spec.metadata["rubygems_mfa_required"] = "true"
 end

data/lib/ietf/data/importer/cli.rb

@@ -1,45 +1,36 @@
 # frozen_string_literal: true
 
 require "thor"
-require "yaml"
 require "fileutils"
-require_relative "group_collection"
-require_relative "scrapers"
 
 module Ietf
   module Data
     module Importer
-      # Command-line interface for IETF/IRTF group data
       class Cli < Thor
-        desc "fetch OUTPUT_FILE", "Fetch IETF/IRTF groups and save to YAML file"
-        option :format, type: :string, default: "yaml", desc: "Output format (yaml or json)"
+        desc "fetch OUTPUT_FILE", "Fetch IETF/IRTF groups and save to file"
+        option :format, type: :string, default: "yaml",
+                        desc: "Output format (yaml or json)"
         def fetch(output_file = nil)
           output_file ||= "ietf_groups.#{options[:format]}"
 
-          # Fetch all groups using the scrapers
-          collection = Ietf::Data::Importer::Scrapers.fetch_all
+          collection = Scrapers.fetch_all
+          collection.save(output_file, format: options[:format].to_sym)
 
-          # Save to file in the requested format
-          format = options[:format].to_sym
-          Ietf::Data::Importer::Scrapers.save_to_file(collection, output_file, format)
+          puts "Saved #{collection.size} groups to #{output_file}"
         end
 
         desc "integrate YAML_FILE", "Integrate YAML file as gem data"
         def integrate(yaml_file)
-          # Validate YAML file
-          begin
-            collection = Ietf::Data::Importer::GroupCollection.from_yaml(File.read(yaml_file))
-          rescue => e
-            puts "Error reading YAML file: #{e.message}"
-            exit 1
-          end
+          collection = GroupCollection.from_yaml(File.read(yaml_file))
 
-          # Save as YAML for gem usage
-          target_yaml = File.join(File.dirname(__FILE__), "groups.yaml")
-          FileUtils.mkdir_p(File.dirname(target_yaml))
-          File.write(target_yaml, File.read(yaml_file))
+          target = File.join(__dir__, "groups.yaml")
+          FileUtils.mkdir_p(File.dirname(target))
+          File.write(target, File.read(yaml_file))
 
-          puts "Integrated #{collection.groups.size} groups into gem"
+          puts "Integrated #{collection.size} groups into gem"
+        rescue StandardError => e
+          puts "Error reading YAML file: #{e.message}"
+          exit 1
         end
       end
     end

data/lib/ietf/data/importer/group.rb

@@ -5,14 +5,17 @@ require "lutaml/model"
 module Ietf
   module Data
     module Importer
-      # Represents a single IETF or IRTF group
       class Group < Lutaml::Model::Serializable
+        ORGANIZATIONS = %w[ietf irtf].freeze
+        STATUSES = %w[active concluded bof proposed].freeze
+        TYPES = %w[wg rg area team program dir ag bof].freeze
+
         attribute :abbreviation, :string
         attribute :name, :string
-        attribute :organization, :string  # 'ietf' or 'irtf'
-        attribute :type, :string  # 'wg', 'rg', etc.
+        attribute :organization, :string
+        attribute :type, :string
         attribute :area, :string
-        attribute :status, :string, values: %w[active concluded bof proposed]
+        attribute :status, :string
         attribute :description, :string
         attribute :chairs, :string, collection: true
         attribute :mailing_list, :string
@@ -36,6 +39,38 @@ module Ietf
           map "charter_url", to: :charter_url
           map "concluded_date", to: :concluded_date
         end
+
+        def active?
+          status == "active"
+        end
+
+        def concluded?
+          status == "concluded"
+        end
+
+        def bof?
+          status == "bof"
+        end
+
+        def proposed?
+          status == "proposed"
+        end
+
+        def ietf?
+          organization == "ietf"
+        end
+
+        def irtf?
+          organization == "irtf"
+        end
+
+        def working_group?
+          type == "wg"
+        end
+
+        def research_group?
+          type == "rg"
+        end
       end
     end
   end

data/lib/ietf/data/importer/group_collection.rb

@@ -6,13 +6,113 @@ require_relative "group"
 module Ietf
   module Data
     module Importer
-      # Represents a collection of IETF and IRTF groups
       class GroupCollection < Lutaml::Model::Serializable
+        include Enumerable
+
         attribute :groups, Group, collection: true
 
         key_value do
           map "groups", to: :groups
         end
+
+        def each(&block)
+          groups.each(&block)
+        end
+
+        def size
+          groups.size
+        end
+
+        def empty?
+          groups.empty?
+        end
+
+        def [](abbreviation)
+          find_by_abbreviation(abbreviation)
+        end
+
+        def find_by_abbreviation(abbreviation)
+          groups.find do |g|
+            g.abbreviation.downcase == abbreviation.to_s.downcase
+          end
+        end
+
+        def exists?(abbreviation)
+          !find_by_abbreviation(abbreviation).nil?
+        end
+
+        def by_organization(org)
+          self.class.new(groups: groups.select do |g|
+            g.organization == org.to_s
+          end)
+        end
+
+        def by_type(type)
+          self.class.new(groups: groups.select do |g|
+            g.type&.downcase == type.to_s.downcase
+          end)
+        end
+
+        def by_area(area)
+          self.class.new(groups: groups.select do |g|
+            g.area&.downcase == area.to_s.downcase
+          end)
+        end
+
+        def active
+          self.class.new(groups: groups.select(&:active?))
+        end
+
+        def concluded
+          self.class.new(groups: groups.select(&:concluded?))
+        end
+
+        def working_groups
+          by_type("wg")
+        end
+
+        def research_groups
+          by_type("rg")
+        end
+
+        def ietf_groups
+          by_organization("ietf")
+        end
+
+        def irtf_groups
+          by_organization("irtf")
+        end
+
+        def group_types
+          groups.filter_map(&:type).uniq.sort
+        end
+
+        def areas
+          groups.filter_map(&:area).uniq.sort
+        end
+
+        def merge(other)
+          self.class.new(groups: groups + other.groups)
+        end
+
+        def self.from_file(path, format: :yaml)
+          return new(groups: []) unless File.exist?(path)
+
+          content = File.read(path)
+          case format
+          when :yaml then from_yaml(content)
+          when :json then from_json(content)
+          else raise ArgumentError, "Unsupported format: #{format}"
+          end
+        end
+
+        def save(path, format: :yaml)
+          case format
+          when :yaml then File.write(path, to_yaml)
+          when :json then File.write(path, to_json)
+          else raise ArgumentError, "Unsupported format: #{format}"
+          end
+        end
       end
     end
   end

data/lib/ietf/data/importer/scrapers/base_scraper.rb

@@ -2,30 +2,39 @@
 
 require "nokogiri"
 require "open-uri"
+require_relative "../group"
+require_relative "../group_collection"
 
 module Ietf
   module Data
     module Importer
       module Scrapers
-        # Base class for web scrapers
         class BaseScraper
-          # Fetch HTML content from a URL and parse it with Nokogiri
-          # @param url [String] The URL to fetch
-          # @return [Nokogiri::HTML::Document] The parsed HTML document
+          def fetch
+            raise NotImplementedError, "#{self.class}#fetch must be implemented"
+          end
+
           def fetch_html(url)
             Nokogiri::HTML(URI.open(url))
-          rescue => e
-            puts "  Error fetching URL #{url}: #{e.message}"
+          rescue StandardError => e
+            log "Error fetching URL #{url}: #{e.message}"
             nil
           end
 
-          # Log a message with indentation
-          # @param message [String] The message to log
-          # @param level [Integer] The indentation level (default: 0)
           def log(message, level = 0)
             indent = "  " * level
             puts "#{indent}#{message}"
           end
+
+          private
+
+          def build_group(attributes)
+            Group.new(attributes)
+          end
+
+          def build_collection(groups)
+            GroupCollection.new(groups: groups)
+          end
         end
       end
     end