ietf-data-importer 0.3.0

data/lib/ietf/data/importer/scrapers.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require_relative "scrapers/base_scraper"
+require_relative "scrapers/ietf_scraper"
+require_relative "scrapers/irtf_scraper"
+require_relative "group_collection"
+
+module Ietf
+  module Data
+    module Importer
+      # Module for IETF/IRTF web scrapers
+      module Scrapers
+        # Fetch all IETF and IRTF groups
+        # @return [Ietf::Data::Importer::GroupCollection] Collection of all groups
+        def self.fetch_all
+          puts "Starting to fetch IETF and IRTF group data..."
+
+          # Fetch IETF groups
+          ietf_groups = IetfScraper.new.fetch
+          puts "Fetched #{ietf_groups.size} IETF groups"
+
+          # Fetch IRTF groups
+          irtf_groups = IrtfScraper.new.fetch
+          puts "Fetched #{irtf_groups.size} IRTF groups"
+
+          # Combine all groups and return as a collection
+          all_groups = ietf_groups + irtf_groups
+          puts "Total: #{all_groups.size} groups"
+
+          Importer::GroupCollection.new(groups: all_groups)
+        end
+
+        # Fetch IETF groups only
+        # @return [Array<Ietf::Data::Importer::Group>] Array of IETF groups
+        def self.fetch_ietf
+          IetfScraper.new.fetch
+        end
+
+        # Fetch IRTF groups only
+        # @return [Array<Ietf::Data::Importer::Group>] Array of IRTF groups
+        def self.fetch_irtf
+          IrtfScraper.new.fetch
+        end
+
+        # Save group collection to a file
+        # @param collection [Ietf::Data::Importer::GroupCollection] Group collection to save
+        # @param file_path [String] Path to the output file
+        # @param format [Symbol] Output format (:yaml or :json)
+        def self.save_to_file(collection, file_path, format = :yaml)
+          case format.to_sym
+          when :yaml
+            File.write(file_path, collection.to_yaml)
+          when :json
+            File.write(file_path, collection.to_json)
+          else
+            raise ArgumentError, "Unsupported format: #{format}"
+          end
+
+          puts "Saved #{collection.groups.size} groups to #{file_path}"
+        end
+      end
+    end
+  end
+end

data/lib/ietf/data/importer/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Ietf
+  module Data
+    module Importer
+      VERSION = "0.3.0"
+    end
+  end
+end

data/lib/ietf/data/importer.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "yaml"
+require_relative "importer/version"
+require_relative "importer/group_collection"
+require_relative "importer/scrapers"
+require_relative "importer/cli"
+
+module Ietf
+  module Data
+    # Main module for IETF/IRTF group data importer
+    module Importer
+      class Error < StandardError; end
+
+      # Path to the groups data file
+      GROUPS_PATH = File.join(File.dirname(__FILE__), "importer", "groups.yaml")
+
+      # Load the groups if the file exists, otherwise return empty collection
+      def self.load_groups
+        if File.exist?(GROUPS_PATH)
+          GroupCollection.from_yaml(File.read(GROUPS_PATH))
+        else
+          GroupCollection.new(groups: [])
+        end
+      end
+
+      # All available groups
+      def self.groups
+        load_groups.groups
+      end
+
+      # Check if a group exists by abbreviation
+      def self.group_exists?(abbreviation)
+        !find_group(abbreviation).nil?
+      end
+
+      # Find a group by its abbreviation (case insensitive)
+      def self.find_group(abbreviation)
+        groups.find { |g| g.abbreviation.downcase == abbreviation.to_s.downcase }
+      end
+
+      # Get all IETF groups
+      def self.ietf_groups
+        groups.select { |g| g.organization == "ietf" }
+      end
+
+      # Get all IRTF groups
+      def self.irtf_groups
+        groups.select { |g| g.organization == "irtf" }
+      end
+
+      # Get all working groups (IETF)
+      def self.working_groups
+        groups.select { |g| g.type == "wg" }
+      end
+
+      # Get all research groups (IRTF)
+      def self.research_groups
+        groups.select { |g| g.type == "rg" }
+      end
+
+      # Get groups by type
+      def self.groups_by_type(type)
+        groups.select { |g| g.type.downcase == type.to_s.downcase }
+      end
+
+      # Get groups by area
+      def self.groups_by_area(area)
+        groups.select { |g| g.area&.downcase == area.to_s.downcase }
+      end
+
+      # Get active groups
+      def self.active_groups
+        groups.select { |g| g.status == "active" }
+      end
+
+      # Get concluded groups
+      def self.concluded_groups
+        groups.select { |g| g.status == "concluded" }
+      end
+
+      # Get all available group types
+      def self.group_types
+        groups.map(&:type).uniq.sort
+      end
+
+      # Get all available areas
+      def self.areas
+        groups.map(&:area).compact.uniq.sort
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification
+name: ietf-data-importer
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Ribose Inc.
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2025-03-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: lutaml-model
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+- !ruby/object:Gem::Dependency
+  name: yaml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |
+  ietf-data-importer offers reliable offline access to metadata for IETF working groups
+  and IRTF research groups. This provides a dependable alternative to the official
+  resources at https://tools.ietf.org/wg/ and https://irtf.org/groups, which may
+  experience downtime or connectivity issues.
+email:
+- open.source@ribose.com
+executables:
+- ietf-data-importer
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/check_update.yml"
+- ".github/workflows/rake.yml"
+- ".github/workflows/release.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- README.adoc
+- Rakefile
+- bin/console
+- bin/setup
+- exe/ietf-data-importer
+- ietf-data-importer.gemspec
+- lib/ietf/data/importer.rb
+- lib/ietf/data/importer/cli.rb
+- lib/ietf/data/importer/group.rb
+- lib/ietf/data/importer/group_collection.rb
+- lib/ietf/data/importer/groups.yaml
+- lib/ietf/data/importer/scrapers.rb
+- lib/ietf/data/importer/scrapers/base_scraper.rb
+- lib/ietf/data/importer/scrapers/ietf_scraper.rb
+- lib/ietf/data/importer/scrapers/irtf_scraper.rb
+- lib/ietf/data/importer/version.rb
+homepage: https://github.com/metanorma/ietf-data-importer
+licenses:
+- BSD-2-Clause
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Offline access to IETF working groups and IRTF research groups metadata
+test_files: []