ietf-data-importer 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0df7ab18290351d1560ddc5e0f0e1e5d976028045f22b6f4b87b89f25f747ffc
+  data.tar.gz: c814b6d33bced977dd20c9c0c3f0f8553007a095eb53116f6c740238eb29a870
+SHA512:
+  metadata.gz: 9b9d8d45a4d9997c0e8f3118c61140f8986bdb169d2379d42f7aee0ee8ef25f6b957db4f66531825a2aac415f940eb1f66b4f4d85c1d89b70fc66045b91e1018
+  data.tar.gz: aa884d901083552f00b086908bbfcc269a5b71d971d493d4333291b9ba02c774441dfb1e6c9b54eff4689ac0656624969e7fdb98f5b1716e5c8275635cea2342

data/.github/workflows/check_update.yml

@@ -0,0 +1,44 @@
+name: check_update
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  check_update:
+    name: Check workgroups update
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '2.6'
+          bundler-cache: true
+
+      - id: update_json
+        run: |
+          bundle exec bin/wg2json.rb > lib/metanorma/ietf/data/workgroups.json
+          git diff-index --quiet HEAD -- && has_changes="false" || has_changes="true"
+          echo "::set-output name=has_changes::${has_changes}"
+
+      - if: ${{ !steps.update_json.outputs.has_changes == 'true' }}
+        run: gem bump --version patch --tag --push
+
+      - if: ${{ !steps.update_json.outputs.has_changes == 'true' }}
+        name: publish to rubygems.org
+        env:
+          RUBYGEMS_API_KEY: ${{secrets.METANORMA_CI_RUBYGEMS_API_KEY}}
+        run: |
+          gem install gem-release
+          envsubst << 'EOF' > ~/.gem/credentials
+          ---
+          :rubygems_api_key: ${RUBYGEMS_API_KEY}
+          EOF
+          chmod 0600 ~/.gem/credentials
+          gem release

data/.github/workflows/rake.yml

@@ -0,0 +1,15 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
+name: rake
+
+on:
+  push:
+    branches: [ master, main ]
+    tags: [ v* ]
+  pull_request:
+
+jobs:
+  rake:
+    uses: metanorma/ci/.github/workflows/generic-rake.yml@main
+    secrets:
+      pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}

data/.github/workflows/release.yml

@@ -0,0 +1,25 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
+name: release
+
+on:
+  workflow_dispatch:
+    inputs:
+      next_version:
+        description: |
+          Next release version. Possible values: x.y.z, major, minor, patch (or pre|rc|etc).
+          Also, you can pass 'skip' to skip 'git tag' and do 'gem push' for the current version
+        required: true
+        default: 'skip'
+  repository_dispatch:
+    types: [ do-release ]
+
+jobs:
+  release:
+    uses: metanorma/ci/.github/workflows/rubygems-release.yml@main
+    with:
+      next_version: ${{ github.event.inputs.next_version }}
+    secrets:
+      rubygems-api-key: ${{ secrets.METANORMA_CI_RUBYGEMS_API_KEY }}
+      pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}
+

data/.gitignore

@@ -0,0 +1,2 @@
+/.rubocop-https*
+/Gemfile.lock

data/.rubocop.yml

@@ -0,0 +1,10 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
+inherit_from:
+  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
+
+# local repo-specific modifications
+# ...
+
+AllCops:
+  TargetRubyVersion: 2.5

data/Gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec
+
+gem "rake"
+
+gem "rspec"
+
+gem "rubocop"

data/README.adoc

@@ -0,0 +1,220 @@
+= IETF Data Importer
+
+image:https://img.shields.io/gem/v/ietf-data-importer.svg["Gem Version", link="https://rubygems.org/gems/ietf-data-importer"]
+image:https://github.com/metanorma/ietf-data-importer/actions/workflows/rake.yml/badge.svg["Build Status", link="https://github.com/metanorma/ietf-data-importer/actions/workflows/rake.yml"]
+image:https://img.shields.io/github/issues-pr-raw/metanorma/ietf-data-importer.svg["Pull Requests", link="https://github.com/metanorma/ietf-data-importer/pulls"]
+image:https://img.shields.io/github/commits-since/metanorma/ietf-data-importer/latest.svg["Commits since latest", link="https://github.com/metanorma/ietf-data-importer/releases"]
+
+== Purpose
+
+IETF Data Importer is a Ruby gem providing access to information about IETF working groups and IRTF research groups.
+
+It includes:
+
+* Data models for IETF and IRTF groups
+* Command-line tools for fetching and updating group data
+* A Ruby API for accessing group information
+
+This gem exists because the official sources often change their layout or may be unavailable:
+
+* https://datatracker.ietf.org/group/ (formerly tools.ietf.org/wg)
+* https://irtf.org/groups
+
+== Migration from metanorma-ietf-data
+
+This gem is the renamed and restructured version of `metanorma-ietf-data`. The namespace and file structure have been changed to match other Metanorma data importer gems.
+
+To migrate from metanorma-ietf-data:
+
+. Replace the following:
++
+[source,diff]
+----
+- require "metanorma/ietf/data"
+- groups = Metanorma::Ietf::Data.groups
++ require "ietf/data/importer"
++ groups = Ietf::Data::Importer.groups
+----
+
+. Update your Gemfile:
++
+[source,diff]
+----
+- gem 'metanorma-ietf-data'
++ gem 'ietf-data-importer'
+----
+
+== Installation
+
+Add this line to your application's Gemfile:
+
+[source,ruby]
+----
+gem 'ietf-data-importer'
+----
+
+And then execute:
+
+[source,shell]
+----
+$ bundle install
+----
+
+Or install it yourself as:
+
+[source,shell]
+----
+$ gem install ietf-data-importer
+----
+
+== Usage
+
+=== For YAML/JSON data users
+
+The gem provides a command-line tool to fetch current group data:
+
+[source,shell]
+----
+$ ietf-data-importer fetch output.yaml
+$ ietf-data-importer fetch output.json --format=json
+----
+
+=== For Ruby API users
+
+[source,ruby]
+----
+require 'ietf/data/importer'
+
+# Check if a group exists
+Ietf::Data::Importer.group_exists?('httpbis')
+
+# Get information about a specific group
+group = Ietf::Data::Importer.find_group('httpbis')
+puts group.name
+puts group.organization
+puts group.status
+
+# Get all IETF working groups
+wgs = Ietf::Data::Importer.working_groups
+
+# Get all IRTF research groups
+rgs = Ietf::Data::Importer.research_groups
+
+# Get groups by a specific type
+type_groups = Ietf::Data::Importer.groups_by_type('wg')
+
+# Get groups by area
+area_groups = Ietf::Data::Importer.groups_by_area('Applications and Real-Time Area')
+
+# Get active or concluded groups
+active = Ietf::Data::Importer.active_groups
+concluded = Ietf::Data::Importer.concluded_groups
+
+# Get all available group types
+types = Ietf::Data::Importer.group_types
+
+# Get all areas
+areas = Ietf::Data::Importer.areas
+----
+
+== Data model
+
+=== Organizations
+
+[options="header"]
+|===
+| Organization | Abbreviation | Description
+| Internet Engineering Task Force | ietf | Standards development organization
+| Internet Research Task Force | irtf | Research organization
+|===
+
+=== Group types
+
+[options="header"]
+|===
+| Type | Abbreviation | Organization | Description
+| Working Group | wg | ietf | IETF working groups
+| Research Group | rg | irtf | IRTF research groups
+| Area | area | ietf | IETF areas
+| Team | team | ietf | IETF administrative teams
+| Program | program | irtf | IRTF programs
+| Directorate | dir | ietf | IETF directorates
+| Advisory Group | ag | ietf/irtf | Advisory groups
+|===
+
+=== Status values
+
+[options="header"]
+|===
+| Status | Description
+| active | Group is currently active
+| concluded | Group has completed its work
+| bof | Birds of a Feather (BOF) session
+| proposed | Proposed but not yet approved
+|===
+
+== Command-line interface
+
+The gem provides a command-line tool with the following commands:
+
+[source,shell]
+----
+# Fetch current IETF/IRTF group data
+$ ietf-data-importer fetch [OUTPUT_FILE] --format=yaml|json
+
+# Integrate a YAML file into the gem for distribution
+$ ietf-data-importer integrate YAML_FILE
+----
+
+== Data schema
+
+The YAML/JSON data follows this schema:
+
+[source,yaml]
+----
+groups:
+- abbreviation: string       # Group abbreviation or acronym
+  name: string               # Full group name
+  organization: string       # 'ietf' or 'irtf'
+  type: string               # Group type (see table above)
+  area: string               # Area name (for IETF WGs)
+  status: string             # Group status
+  description: string        # Description or charter text
+  chairs:                    # Array of chairs
+    - string                 # Chair name
+  mailing_list: string       # Mailing list address
+  mailing_list_archive: string # Archive URL
+  website_url: string        # Group website URL
+  charter_url: string        # Charter document URL
+  concluded_date: string     # ISO date of conclusion (if applicable)
+----
+
+[example]
+====
+[source,yaml]
+----
+groups:
+- abbreviation: httpbis
+  name: HTTP
+  organization: ietf
+  type: wg
+  area: Applications and Real-Time Area
+  status: active
+  description: The HTTP working group is chartered to maintain and develop the Hypertext Transfer Protocol...
+  chairs:
+    - Chair Person 1
+    - Chair Person 2
+  mailing_list: httpbis@ietf.org
+  mailing_list_archive: https://mailarchive.ietf.org/arch/browse/httpbis/
+  website_url: https://httpwg.org/
+  charter_url: https://datatracker.ietf.org/wg/httpbis/about/
+----
+====
+
+== Copyright
+
+This gem is developed, maintained and funded by https://www.ribose.com[Ribose Inc.]
+
+== License
+
+The gem is available as open source under the terms of the https://opensource.org/licenses/BSD-2-Clause[2-Clause BSD License].

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "ietf/data/importer"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/ietf-data-importer

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../lib/ietf/data/importer/cli"
+
+Ietf::Data::Importer::Cli.start(ARGV)

data/ietf-data-importer.gemspec

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "lib/ietf/data/importer/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "ietf-data-importer"
+  spec.version       = Ietf::Data::Importer::VERSION
+  spec.authors       = ["Ribose Inc."]
+  spec.email         = ["open.source@ribose.com"]
+
+  spec.summary = <<~SUMMARY
+    Offline access to IETF working groups and IRTF research groups metadata
+  SUMMARY
+  spec.description = <<~DESCRIPTION
+    ietf-data-importer offers reliable offline access to metadata for IETF working groups
+    and IRTF research groups. This provides a dependable alternative to the official
+    resources at https://tools.ietf.org/wg/ and https://irtf.org/groups, which may
+    experience downtime or connectivity issues.
+  DESCRIPTION
+  spec.homepage      = "https://github.com/metanorma/ietf-data-importer"
+  spec.license       = "BSD-2-Clause"
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0")
+
+  spec.add_dependency "lutaml-model", "~> 0.7"
+  spec.add_dependency "thor", "~> 1.0"
+  spec.add_dependency "nokogiri", "~> 1.18"
+  spec.add_dependency "yaml"
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{\A(?:test|spec|features)/})
+    end
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+end

data/lib/ietf/data/importer/cli.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "thor"
+require "yaml"
+require "fileutils"
+require_relative "group_collection"
+require_relative "scrapers"
+
+module Ietf
+  module Data
+    module Importer
+      # Command-line interface for IETF/IRTF group data
+      class Cli < Thor
+        desc "fetch OUTPUT_FILE", "Fetch IETF/IRTF groups and save to YAML file"
+        option :format, type: :string, default: "yaml", desc: "Output format (yaml or json)"
+        def fetch(output_file = nil)
+          output_file ||= "ietf_groups.#{options[:format]}"
+
+          # Fetch all groups using the scrapers
+          collection = Ietf::Data::Importer::Scrapers.fetch_all
+
+          # Save to file in the requested format
+          format = options[:format].to_sym
+          Ietf::Data::Importer::Scrapers.save_to_file(collection, output_file, format)
+        end
+
+        desc "integrate YAML_FILE", "Integrate YAML file as gem data"
+        def integrate(yaml_file)
+          # Validate YAML file
+          begin
+            collection = Ietf::Data::Importer::GroupCollection.from_yaml(File.read(yaml_file))
+          rescue => e
+            puts "Error reading YAML file: #{e.message}"
+            exit 1
+          end
+
+          # Save as YAML for gem usage
+          target_yaml = File.join(File.dirname(__FILE__), "groups.yaml")
+          FileUtils.mkdir_p(File.dirname(target_yaml))
+          File.write(target_yaml, File.read(yaml_file))
+
+          puts "Integrated #{collection.groups.size} groups into gem"
+        end
+      end
+    end
+  end
+end

data/lib/ietf/data/importer/group.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ietf
+  module Data
+    module Importer
+      # Represents a single IETF or IRTF group
+      class Group < Lutaml::Model::Serializable
+        attribute :abbreviation, :string
+        attribute :name, :string
+        attribute :organization, :string  # 'ietf' or 'irtf'
+        attribute :type, :string  # 'wg', 'rg', etc.
+        attribute :area, :string
+        attribute :status, :string, values: %w[active concluded bof proposed]
+        attribute :description, :string
+        attribute :chairs, :string, collection: true
+        attribute :mailing_list, :string
+        attribute :mailing_list_archive, :string
+        attribute :website_url, :string
+        attribute :charter_url, :string
+        attribute :concluded_date, :date
+
+        key_value do
+          map "abbreviation", to: :abbreviation
+          map "name", to: :name
+          map "organization", to: :organization
+          map "type", to: :type
+          map "area", to: :area
+          map "status", to: :status
+          map "description", to: :description
+          map "chairs", to: :chairs
+          map "mailing_list", to: :mailing_list
+          map "mailing_list_archive", to: :mailing_list_archive
+          map "website_url", to: :website_url
+          map "charter_url", to: :charter_url
+          map "concluded_date", to: :concluded_date
+        end
+      end
+    end
+  end
+end

data/lib/ietf/data/importer/group_collection.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+require_relative "group"
+
+module Ietf
+  module Data
+    module Importer
+      # Represents a collection of IETF and IRTF groups
+      class GroupCollection < Lutaml::Model::Serializable
+        attribute :groups, Group, collection: true
+
+        key_value do
+          map "groups", to: :groups
+        end
+      end
+    end
+  end
+end