relaton-itu 1.14.0 → 1.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ef87fd606902663b9e0236740fabc892089c45496363a7e2130d5575592a0e07
-  data.tar.gz: f283ad42e01ffc75850880b4bb224e601db7db1c3df71fe883800c1b4ba3d382
+  metadata.gz: 6766812ad6660d739d3d4e4c1778846119eaca063c3c5cfc34242ffca6414a6b
+  data.tar.gz: 7c3024fc052ddd75a7c82ab1b3231834d6b0c964cf940f5fa1e2c28c0ff5c085
 SHA512:
-  metadata.gz: 99e6637a4eaa1121993c22ff14a4c3dc5a2e00a875b86ee8187b39d81025ac28d3771dca370e9f2c5b2f42be9e77c37e8a4b9576952c5facecf0b407d5e730b5
-  data.tar.gz: 57e935f3354bf931bcb5cf6ad851b113139ef25100e9e78a8d8766d193dfbb002c801daa698807e5e2844c43b69128a0cbdef1c411de2599dcc5fc028733932f
+  metadata.gz: 76647371a1ef4b6eaf3a4f57bd8bff607dda86f63f0f928e8969d9e9a25532eb456f7781740068f094b1fa4d9b505092ff7710e5fb16b9bdbd168207abc70f23
+  data.tar.gz: d87e9793f3802bde3d39d672c7c10900f5fd1298811bf6bb0fa129f577020c5e3cbae9d52ef5623e6d3e7933c322b2cdc555e824830cfa4707e48169fe92cd73

data/.github/workflows/rake.yml

@@ -5,6 +5,7 @@ name: rake
 on:
   push:
     branches: [ master, main ]
+    tags: [ v* ]
   pull_request:
 
 jobs:

data/.github/workflows/release.yml

@@ -10,8 +10,9 @@ on:
           Next release version. Possible values: x.y.z, major, minor, patch or pre|rc|etc
         required: true
         default: 'skip'
-  push:
-    tags: [ v* ]
+  repository_dispatch:
+    types: [ do-release ]
+
 
 jobs:
   release:

data/.gitignore

@@ -7,6 +7,7 @@
 /spec/reports/
 /tmp/
 .vscode/
+/vendor/
 .rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
 
 # rspec failure tracking

data/.rubocop.yml

@@ -7,6 +7,6 @@ require: rubocop-rails
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 2.7
 Rails:
   Enabled: false

data/Gemfile

@@ -4,3 +4,12 @@ Encoding.default_internal = Encoding::UTF_8
 source "https://rubygems.org"
 # Specify your gem's dependencies in relaton_itu.gemspec
 gemspec
+
+gem "equivalent-xml", "~> 0.6"
+gem "pry-byebug"
+gem "rake", "~> 13.0"
+gem "rspec", "~> 3.0"
+gem "ruby-jing"
+gem "simplecov"
+gem "vcr"
+gem "webmock"

data/README.adoc

@@ -69,7 +69,7 @@ item.to_xml bibdata: true
     </bibdata>"
 ----
 
-=== Get code, and year
+=== Get document by code and year
 [source,ruby]
 ----
 RelatonItu::ItuBibliography.get("ITU-T L.163", "2018", {})
@@ -123,7 +123,7 @@ item.link
  #<RelatonBib::TypedUri:0x00007f82d665f0f8 @content=#<Addressable::URI:0xc184 URI:https://www.itu.inthttp//handle.itu.int/11.1002/1000/13786-en?locatt=format:pdf&auth>, @type="obp">]
 ----
 
-=== Create bibliographic item from YAML
+=== Create a bibliographic item from YAML
 [source,ruby]
 ----
 hash = YAML.load_file 'spec/examples/itu_bib_item.yml'
@@ -135,6 +135,25 @@ RelatonItu::ItuBibliographicItem.from_hash hash
 ...
 ----
 
+=== Fetch data
+
+This gem uses the (https://extranet.itu.int/brdocsearch) dataset as a data source. +
+
+The method `RelatonItu::DataFetcher.fetch(output: "data", format: "yaml")` fetches all the documents from the dataset and saves them to the `./data` folder in YAML format.
+Arguments:
+
+- `output` - folder to save documents (default 'data').
+- `format` - the format in which the documents are saved. Possible formats are: `yaml`, `xml`, `bibxxml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonItu::DataFetcher.fetch output: "dir", format: "xml"
+Started at: 2023-05-27 09:21:16 -0400
+Stopped at: 2023-05-27 09:27:45 -0400
+Done in: 390 sec.
+=> nil
+----
+
 == Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/grammars/basicdoc.rng

@@ -522,7 +522,6 @@
       <value>tip</value>
       <value>important</value>
       <value>caution</value>
-      <value>statement</value>
     </choice>
   </define>
   <define name="figure">

data/grammars/biblio.rng

@@ -216,6 +216,9 @@
       <optional>
         <ref name="fullname"/>
       </optional>
+      <zeroOrMore>
+        <ref name="credential"/>
+      </zeroOrMore>
       <zeroOrMore>
         <ref name="affiliation"/>
       </zeroOrMore>
@@ -232,6 +235,11 @@
       <ref name="FullNameType"/>
     </element>
   </define>
+  <define name="credential">
+    <element name="credential">
+      <text/>
+    </element>
+  </define>
   <define name="FullNameType">
     <choice>
       <group>
@@ -305,7 +313,9 @@
       <zeroOrMore>
         <ref name="affiliationdescription"/>
       </zeroOrMore>
-      <ref name="organization"/>
+      <optional>
+        <ref name="organization"/>
+      </optional>
     </element>
   </define>
   <define name="affiliationname">
@@ -1316,7 +1326,7 @@
       <value>commentaryOf</value>
       <value>hasCommentary</value>
       <value>related</value>
-      <value>complements</value>
+      <value>hasComplement</value>
       <value>complementOf</value>
       <value>obsoletes</value>
       <value>obsoletedBy</value>

data/lib/relaton_itu/data_fetcher.rb

@@ -0,0 +1,147 @@
+module RelatonItu
+  class DataFetcher
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub "bibxml", "xml"
+    end
+
+    def files
+      @files ||= []
+    end
+
+    def index
+      @index ||= Relaton::Index.find_or_create :itu, file: "index-v1.yaml"
+    end
+
+    # @return agent [Mechanize]
+    def agent
+      @agent ||= Mechanize.new
+    end
+
+    # @return workers [RelatonBib::WorkersPool]
+    def workers
+      return @workers if @workers
+
+      @workers = RelatonBib::WorkersPool.new 10
+      @workers.tap do |w|
+        w.worker { |row| parse_page(*row) }
+      end
+    end
+
+    # @param url [String]
+    # @param type [String]
+    def parse_page(url, type)
+      doc = agent.get url
+      bib = DataParserR.parse doc, url, type
+      write_file bib
+    rescue => e # rubocop:disable Style/RescueStandardError
+      warn e.message
+      warn e.backtrace
+    end
+
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    def fetch
+      fetch_recommendation
+      fetch_question
+      fetch_report
+      fetch_handbook
+      fetch_resolution
+      workers.end
+      workers.result
+      index.save
+    end
+
+    def fetch_recommendation
+      url = "https://extranet.itu.int/brdocsearch/_layouts/15/inplview.aspx?" \
+            "List=%7B0661B581-2413-4E84-BAB2-77E6DB27AF7F%7D&" \
+            "View=%7BC81191DD-48C4-4881-9CB7-FB61C683FE98%7D&" \
+            "ViewCount=123&" \
+            "IsXslView=TRUE&" \
+            "IsCSR=TRUE&" \
+            "ListViewPageUrl=https%3A%2F%2Fextranet.itu.int%2Fbrdocsearch%2FR-REC%2FForms%2Ffolders_inforce.aspx&" \
+            "FolderCTID=0x012001"
+      json_index url, "recommendation"
+    end
+
+    def fetch_question
+      url = "https://extranet.itu.int/brdocsearch/R-QUE/Forms/folders_inforce.aspx"
+      html_index url, "question"
+    end
+
+    def fetch_report
+      url = "https://extranet.itu.int/brdocsearch/_layouts/15/inplview.aspx?" \
+            "List=%7B82E4A13D-C7F3-4844-9E8A-2463C4B7784F%7D&" \
+            "View=%7B94CC1561-E4AC-4317-B402-AA0AADD7F414%7D&" \
+            "ViewCount=407&" \
+            "IsXslView=TRUE&" \
+            "IsCSR=TRUE&" \
+            "ListViewPageUrl=https%3A%2F%2Fextranet.itu.int%2Fbrdocsearch%2FR-REP%2FForms%2FFolders%2520InForce.aspx&" \
+            "FolderCTID=0x012001"
+      json_index url, "technical-report"
+    end
+
+    def fetch_handbook
+      url = "https://extranet.itu.int/brdocsearch/R-HDB/Forms/Folders%20InForce.aspx"
+      html_index url, "handbook"
+    end
+
+    def fetch_resolution
+      url = "https://extranet.itu.int/brdocsearch/R-RES/Forms/Folders%20InForce.aspx"
+      html_index url, "resolution"
+    end
+
+    # #param url [String]
+    # @param type [String]
+    def json_index(url, type) # rubocop:disable Metrics/AbcSize
+      result = agent.post url
+      json = JSON.parse result.body
+      json["Row"].each { |row| workers << [row["serverurl.progid"].sub(/^1/, ""), type] }
+      return unless json["NextHref"]
+
+      nexturl = url.sub(/(Paged|FolderCTID)=.+/, json["NextHref"].match(/(?<=aspx\?).+/).to_s)
+      json_index nexturl, type
+    end
+
+    # #param url [String]
+    # @param type [String]
+    def html_index(url, type)
+      resp = agent.get url
+      result = Nokogiri::HTML resp.body
+      result.xpath("//table//table/tr[position() > 1]").each do |hit|
+        url = hit.at("td/a")[:onclick].match(%r{https://[^']+}).to_s
+        workers << [url, type]
+      end
+    end
+
+    # @param bib [RelatonItu::ItuBibliographicItem]
+    def write_file(bib) # rubocop:disable Metrics/AbcSize
+      id = bib.docidentifier[0].id.gsub(/[\s.]/, "_")
+      file = "#{@output}/#{id}.#{@ext}"
+      if files.include? file
+        warn "File #{file} exists."
+      else
+        files << file
+      end
+      index.add_or_update bib.docidentifier[0].id, file
+      File.write file, content(bib), encoding: "UTF-8"
+    end
+
+    def content(bib)
+      case @format
+      when "yaml" then bib.to_hash.to_yaml
+      when "xml" then bib.to_xml bibdata: true
+      when "bibxml" then bib.to_bibxml
+      end
+    end
+  end
+end

data/lib/relaton_itu/data_parser_r.rb

@@ -0,0 +1,94 @@
+module RelatonItu
+  module DataParserR
+    extend self
+
+    #
+    # Parse ITU-R document.
+    #
+    # @param [Mechanize::Page] doc mechanize page
+    # @param [String] url document url
+    # @param [String] type document type
+    #
+    # @return [RelatonItu::ItuBibliographicItem] bibliographic item
+    #
+    def parse(doc, url, type)
+      RelatonItu::ItuBibliographicItem.new(
+        docid: fetch_docid(doc), title: fetch_title(doc),
+        abstract: fetch_abstract(doc), date: fetch_date(doc), language: ["en"],
+        link: fetch_link(url), script: ["Latn"], docstatus: fetch_status(doc),
+        type: "standard", doctype: type
+      )
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Araay<RelatonBib::DocumentIdentifier>]
+    def fetch_docid(doc)
+      # id = doc.at('//h3[.="Number"]/parent::td/following-sibling::td[2]').text # .match(/^[^\s\(]+/).to_s
+      # %r{^(?<id1>[^\s\(\/]+(\/\d+)?)(\/(?<id2>\w+[^\s\(]+))?} =~ id
+      id = doc.at('//div[@id="idDocSetPropertiesWebPart"]/h2').text.match(/^R-\w+-([^-]+(?:-\d{1,3})?)/)[1]
+      [RelatonBib::DocumentIdentifier.new(type: "ITU", id: "ITU-R #{id}", primary: true)]
+      # docid << RelatonBib::DocumentIdentifier.new(type: 'ITU', id: id2) if id2
+      # docid
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Araay<RelatonBib::TypedTitleString>]
+    def fetch_title(doc)
+      content = doc.at('//h3[.="Title"]/parent::td/following-sibling::td[2]').text
+      [RelatonBib::TypedTitleString.new(type: "main", content: content, language: "en", script: "Latn")]
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Array<RelatonBib::FormattedString>]
+    def fetch_abstract(doc)
+      doc.xpath('//h3[.="Observation"]/parent::td/following-sibling::td[2]').map do |a|
+        c = a.text.strip
+        RelatonBib::FormattedString.new content: c, language: "en", script: "Latn" unless c.empty?
+      end.compact
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Araay<RelatonBib::BibliographicDate>]
+    def fetch_date(doc)
+      dates = []
+      date = doc.at('//h3[.="Approval_Date"]/parent::td/following-sibling::td[2]',
+                    '//h3[.="Approval date"]/parent::td/following-sibling::td[2]',
+                    '//h3[.="Approval year"]/parent::td/following-sibling::td[2]')
+      dates << parse_date(date.text, "confirmed") if date
+
+      date = doc.at('//h3[.="Version year"]/parent::td/following-sibling::td[2]')
+      dates << parse_date(date.text, "updated") if date
+      date = doc.at('//div[@id="idDocSetPropertiesWebPart"]/h2').text.match(/(?<=-)(19|20)\d{2}/)
+      dates << parse_date(date.to_s, "published") if date
+      dates
+    end
+
+    # @param date [String]
+    # @param type [String]
+    # @return [RelatonBib::BibliographicDate]
+    def parse_date(date, type)
+      d = case date
+          # when /^\d{4}$/ then date
+          when /(\d{4})(\d{2})/ then "#{$1}-#{$2}"
+          when %r{(\d{1,2})/(\d{1,2})/(\d{4})} then "#{$3}-#{$1}-#{$2}"
+          else date
+          end
+      RelatonBib::BibliographicDate.new(type: type, on: d)
+    end
+
+    # @param url [String]
+    # @return [Array<RelatonBib::TypedUri>]
+    def fetch_link(url)
+      [RelatonBib::TypedUri.new(type: "src", content: url)]
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [RelatonBib::DocumentStatus, nil]
+    def fetch_status(doc)
+      s = doc.at('//h3[.="Status"]/parent::td/following-sibling::td[2]')
+      return unless s
+
+      RelatonBib::DocumentStatus.new stage: s.text
+    end
+  end
+end

data/lib/relaton_itu/hit.rb

@@ -8,7 +8,7 @@ module RelatonItu
     # Parse page.
     # @return [RelatonItu::ItuBibliographicItem]
     def fetch
-      @fetch ||= Scrapper.parse_page self, hit_collection.gi_imp
+      @fetch ||= Scrapper.parse_page self, imp: hit_collection.gi_imp
     end
   end
 end

data/lib/relaton_itu/hit_collection.rb

@@ -8,6 +8,8 @@ module RelatonItu
   # Page of hit collection.
   class HitCollection < RelatonBib::HitCollection
     DOMAIN = "https://www.itu.int"
+    GH_ITU_R = "https://raw.githubusercontent.com/relaton/relaton-data-itu-r/master/"
+    INDEX_FILE = "index-v1.yaml"
 
     # @return [TrueClass, FalseClass]
     attr_reader :gi_imp
@@ -23,6 +25,7 @@ module RelatonItu
       @agent = Mechanize.new
       agent.user_agent_alias = "Mac Safari"
       @gi_imp = /\.Imp\d/.match?(ref)
+      @array = []
 
       case ref
       when /^(ITU-T|ITU-R\sRR)/
@@ -35,26 +38,25 @@ module RelatonItu
     private
 
     def request_search
-      url = "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
+      url = "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/RunSearch"
       data = { json: params.to_json }
-      resp = agent.post url, data.to_json, "Content-Type" => "application/json"
+      resp = agent.post url, data
       @array = hits JSON.parse(resp.body)
     end
 
     # @param ref [String] a document ref
-    def request_document(ref) # rubocop:todo Metrics/MethodLength
-      uri = URI::HTTPS.build(
-        host: "raw.githubusercontent.com",
-        path: "/relaton/relaton-data-itu-r/master/data/#{ref.gsub(/[\s.]/, '_')}.yaml",
-      )
+    def request_document(ref) # rubocop:todo Metrics/MethodLength, Metrics/AbcSize
+      index = Relaton::Index.find_or_create :itu, url: "#{GH_ITU_R}index-v1.zip", file: INDEX_FILE
+      row = index.search(ref).min_by { |i| i[:id] }
+      return unless row
+
+      uri = URI("#{GH_ITU_R}#{row[:file]}")
       resp = Net::HTTP.get_response(uri)
-      if resp.code == "404"
-        @array = []
-        return
-      end
+      return if resp.code == "404"
 
       hash = YAML.safe_load resp.body
       item_hash = HashConverter.hash_to_bib(hash)
+      item_hash[:fetched] = Date.today.to_s
       item = ItuBibliographicItem.new(**item_hash)
       hit = Hit.new({ url: uri.to_s }, self)
       hit.fetch = item
@@ -88,7 +90,7 @@ module RelatonItu
               "Selected" => false,
               "Value" => "",
               "Label" => "Name",
-              "Target" => "\\/name_s",
+              "Target" => "/name_s",
               "TypeName" => "CHECKBOX",
               "GetCriteriaType" => 0,
             },
@@ -96,7 +98,7 @@ module RelatonItu
               "Selected" => false,
               "Value" => "",
               "Label" => "Short description",
-              "Target" => "\\/short_description_s",
+              "Target" => "/short_description_s",
               "TypeName" => "CHECKBOX",
               "GetCriteriaType" => 0,
             },
@@ -104,7 +106,7 @@ module RelatonItu
               "Selected" => false,
               "Value" => "",
               "Label" => "File content",
-              "Target" => "\\/file",
+              "Target" => "/file",
               "TypeName" => "CHECKBOX",
               "GetCriteriaType" => 0,
             },
@@ -125,7 +127,7 @@ module RelatonItu
       data["results"].map do |h|
         code  = h["Media"]["Name"]
         title = h["Title"]
-        url   = h["Redirection"]
+        url   = "#{DOMAIN}#{h['Redirection']}"
         type  = h["Collection"]["Group"].downcase[0...-1]
         Hit.new({ code: code, title: title, url: url, type: type }, self)
       end

data/lib/relaton_itu/itu_bibliography.rb

@@ -59,19 +59,19 @@ module RelatonItu
 
       def fetch_ref_err(code, year, missed_years) # rubocop:disable Metrics/MethodLength
         id = year ? "#{code}:#{year}" : code
-        warn "[relaton-itu] WARNING: no match found online for #{id}. "\
-          "The code must be exactly like it is on the standards website."
+        warn "[relaton-itu] WARNING: no match found online for #{id}. " \
+             "The code must be exactly like it is on the standards website."
         unless missed_years.empty?
-          warn "[relaton-itu] (There was no match for #{year}, though there "\
-            "were matches found for #{missed_years.join(', ')}.)"
+          warn "[relaton-itu] (There was no match for #{year}, though there " \
+               "were matches found for #{missed_years.join(', ')}.)"
         end
         if /\d-\d/.match? code
-          warn "[relaton-itu] The provided document part may not exist, or "\
-            "the document may no longer be published in parts."
+          warn "[relaton-itu] The provided document part may not exist, or " \
+               "the document may no longer be published in parts."
         else
-          warn "[relaton-itu] If you wanted to cite all document parts for the reference, "\
-            "use \"#{code} (all parts)\".\nIf the document is not a standard, "\
-            "use its document type abbreviation (TS, TR, PAS, Guide)."
+          warn "[relaton-itu] If you wanted to cite all document parts for the reference, " \
+               "use \"#{code} (all parts)\".\nIf the document is not a standard, " \
+               "use its document type abbreviation (TS, TR, PAS, Guide)."
         end
         nil
       end

data/lib/relaton_itu/processor.rb

@@ -7,6 +7,7 @@ module RelatonItu
       @prefix = "ITU"
       @defaultprefix = %r{^ITU\s}
       @idtype = "ITU"
+      @datasets = %w[itu-r]
     end
 
     # @param code [String]
@@ -17,6 +18,18 @@ module RelatonItu
       ::RelatonItu::ItuBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from https://extranet.itu.int/brdocsearch/
+    #
+    # @param [String] source source name (itu-r)
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents, default is data
+    # @option opts [String] :format output format, default is yaml
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonItu::ItuBibliographicItem]
     def from_xml(xml)
@@ -34,5 +47,12 @@ module RelatonItu
     def grammar_hash
       @grammar_hash ||= ::RelatonItu.grammar_hash
     end
+
+    #
+    # Remove index file
+    #
+    def remove_index_file
+      Relaton::Index.find_or_create(:itu, url: true, file: HitCollection::INDEX_FILE).remove_file
+    end
   end
 end

data/lib/relaton_itu/scrapper.rb

@@ -27,7 +27,7 @@ module RelatonItu
       # Parse page.
       # @param hit [RelatonItu::Hit]
       # @return [Hash]
-      def parse_page(hit, imp = false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def parse_page(hit, imp: false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
         doc = get_page hit
         return unless doc.code == "200"
 
@@ -190,7 +190,7 @@ module RelatonItu
           did = RelatonBib::DocumentIdentifier.new(id: ref.text, type: "ITU")
           bibitem = ItuBibliographicItem.new(formattedref: fref, docid: [did],
                                              type: "standard")
-          { type: "complements", bibitem: bibitem }
+          { type: "complementOf", bibitem: bibitem }
         end
       end
 

data/lib/relaton_itu/version.rb

@@ -1,3 +1,3 @@
 module RelatonItu
-  VERSION = "1.14.0".freeze
+  VERSION = "1.14.2".freeze
 end

data/lib/relaton_itu.rb

@@ -1,7 +1,10 @@
 require "mechanize"
+require "digest/md5"
+require "relaton/index"
 require "relaton_itu/version"
 require "relaton_itu/itu_bibliography"
-require "digest/md5"
+require "relaton_itu/data_fetcher"
+require "relaton_itu/data_parser_r"
 
 module RelatonItu
   class Error < StandardError; end

data/relaton-itu.gemspec

@@ -1,5 +1,4 @@
-
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "relaton_itu/version"
 
@@ -9,10 +8,10 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Ribose Inc."]
   spec.email         = ["open.source@ribose.com"]
 
-  spec.summary       = "RelatonItu: retrieve ITU Standards for bibliographic use "\
-                       "using the BibliographicItem model"
-  spec.description   = "RelatonItu: retrieve ITU Standards for bibliographic use "\
-                       "using the BibliographicItem model"
+  spec.summary       = "RelatonItu: retrieve ITU Standards for bibliographic " \
+                       "use using the BibliographicItem model"
+  spec.description   = "RelatonItu: retrieve ITU Standards for bibliographic " \
+                       "use using the BibliographicItem model"
   spec.homepage      = "https://github.com/metanorma/relaton-itu"
   spec.license       = "MIT"
 
@@ -24,17 +23,9 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
-
-  spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "ruby-jing"
-  spec.add_development_dependency "simplecov"
-  spec.add_development_dependency "vcr", "~> 5.0.0"
-  spec.add_development_dependency "webmock"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
   spec.add_dependency "mechanize", "~> 2.8.0"
   spec.add_dependency "relaton-bib", "~> 1.14.0"
+  spec.add_dependency "relaton-index", "~> 0.2.0"
 end

metadata

@@ -1,155 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: relaton-itu
 version: !ruby/object:Gem::Version
-  version: 1.14.0
+  version: 1.14.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-12-05 00:00:00.000000000 Z
+date: 2023-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: equivalent-xml
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: pry-byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: ruby-jing
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: vcr
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.0
-  type: :development
+        version: 2.8.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.0
-- !ruby/object:Gem::Dependency
-  name: webmock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.8.0
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: 1.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: 1.14.0
 - !ruby/object:Gem::Dependency
-  name: relaton-bib
+  name: relaton-index
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: 0.2.0
 description: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem
   model'
 email:
@@ -176,6 +78,8 @@ files:
 - grammars/relaton-itu-compile.rng
 - grammars/relaton-itu.rng
 - lib/relaton_itu.rb
+- lib/relaton_itu/data_fetcher.rb
+- lib/relaton_itu/data_parser_r.rb
 - lib/relaton_itu/editorial_group.rb
 - lib/relaton_itu/hash_converter.rb
 - lib/relaton_itu/hit.rb
@@ -201,14 +105,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
 summary: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem