relaton-ecma 1.14.0 → 1.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2bfb56317935db58c7697daf05caf958a85995fa3d66533034a18cdeb671a178
-  data.tar.gz: 63d485105638a55c444ceecf185296b159308ca54981f8564cb755422177f79e
+  metadata.gz: ccb93a32e1674146e6e2465d8a264f9322f12616be106f921f6176412ef1e22d
+  data.tar.gz: 163267a6fbffc14dc71eaf072e29856cd899a383055ea21f2126d7cd12675128
 SHA512:
-  metadata.gz: 1b44febabc989d7c41f0561234c5715d83b0e393fa12f7b48ccfd749b20f39a8ffe334a27cef7eb875c4c14d08abe4807dfb098018b31cbb911c8f0b3a487e21
-  data.tar.gz: 878ee9903834c5d832f5a23ffd09a43c6343a990ab91756e7a874866d7a505374e6e922e3ba6c7fd007b848238c6d771072826191d8cb0e1f7c11eb02d3a934c
+  metadata.gz: 56b043e9a77b57410600001a6cfbde0fe24d9b723d3d2911013e3ce933f3fe5c9297d7032e3725efbf150c77b3938cf59904e40cb63a9a3f9eb96a50e0b402f8
+  data.tar.gz: de9fce557065d77645e09d18a41f262b01558cfeb844e3007b46551936ea66261a3a37bc37afe8c6b6ecdbc431d93b0411e98862b93a69644e3d41ab3433f76e

data/.github/workflows/rake.yml

@@ -5,6 +5,7 @@ name: rake
 on:
   push:
     branches: [ master, main ]
+    tags: [ v* ]
   pull_request:
 
 jobs:

data/.github/workflows/release.yml

@@ -10,8 +10,9 @@ on:
           Next release version. Possible values: x.y.z, major, minor, patch or pre|rc|etc
         required: true
         default: 'skip'
-  push:
-    tags: [ v* ]
+  repository_dispatch:
+    types: [ do-release ]
+
 
 jobs:
   release:

data/Gemfile

@@ -5,3 +5,9 @@ gemspec
 
 gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
+
+gem "pry-byebug"
+gem "ruby-jing"
+gem "simplecov"
+gem "vcr"
+gem "webmock"

data/README.adoc

@@ -29,25 +29,67 @@ Or install it yourself as:
 
 == Usage
 
-=== Search document
+=== Fetch documents
+
+Documents can be fetched by reference. The structure of the reference depends on the type of the document. There are three types of documents:
+
+- ECMA standards
+- ECMA technical reports
+- ECMA mementos
+
+ECMA standards have the following reference structure: `ECMA-{NUMBER}[ ed{EDITION}][ vol{VOLUME}]`. Where: `NUMBER` is a number of the standard, `EDITION` is an edition of the standard, and `VOLUME` is a volume of the standard. The `EDITION` and `VOLUME` are optional. If `EDITION` is not specified, the latest edition of the standard will be fetched. If `VOLUME` is not specified, the first volume of the standard will be fetched. +
+
+ECMA technical reports have the following reference structure: `ECMA TR/{NUMBER}[ ed{EDITION}]`. Where: `NUMBER` is a number of the technical report, and `EDITION` is an edition of the technical report. The `EDITION` is optional. If `EDITION` is not specified, the latest edition of the technical report will be fetched. +
+
+ECMA mementos have the following reference structure: `ECMA MEM/{YEAR}`. Where: `YEAR` is an year of the memento.
 
 [source,ruby]
 ----
 require 'relaton_ecma'
 => true
 
+# fetch ECMA standard
 item = RelatonEcma::EcmaBibliography.get 'ECMA-6'
 [relaton-ecma] ("ECMA-6") fetching...
 [relaton-ecma] ("ECMA-6") found ECMA-6
 #<RelatonEcma::BibliographicItem:0x00007fc645b11c10
 ...
 
+# fetch ECMA standard with edition and volume
+RelatonEcma::EcmaBibliography.get "ECMA-269 ed3 vol2"
+[relaton-ecma] ("ECMA-269 ed3 vol2") fetching...
+[relaton-ecma] ("ECMA-269 ed3 vol2") found ECMA-269
+=> #<RelatonEcma::BibliographicItem:0x0000000106ac8210
+...
+
+# fetch the last edition of ECMA standard
+bib = RelatonEcma::EcmaBibliography.get "ECMA-269"
+[relaton-ecma] ("ECMA-269") fetching...
+[relaton-ecma] ("ECMA-269") found ECMA-269
+=> #<RelatonEcma::BibliographicItem:0x000000010a408480
+...
+
+bib.edition.content
+=> "9"
+
+# fetch the first volume of ECMA standard
+bib = RelatonEcma::EcmaBibliography.get "ECMA-269 ed3"
+[relaton-ecma] ("ECMA-269 ed3") fetching...
+[relaton-ecma] ("ECMA-269 ed3") found ECMA-269
+=> #<RelatonEcma::BibliographicItem:0x000000010a3ed0e0
+...
+
+bib.extent.first.reference_from
+=> "1"
+
+# fetch ECMA technical report
 RelatonEcma::EcmaBibliography.get 'ECMA TR/18'
 [relaton-ecma] ("ECMA TR/18") fetching...
 [relaton-ecma] ("ECMA TR/18") found ECMA TR/18
 => #<RelatonEcma::BibliographicItem:0x00007fc645c00cc0
 ...
 
+# fetch ECMA memento
 RelatonEcma::EcmaBibliography.get "ECMA MEM/2021"
 [relaton-ecma] ("ECMA MEM/2021") fetching...
 [relaton-ecma] ("ECMA MEM/2021") found ECMA MEM/2021
@@ -113,6 +155,25 @@ item = RelatonEcma::XMLParser.from_xml File.read("spec/fixtures/bibdata.xml")
 ...
 ----
 
+=== Fetch data
+
+This gem uses a https://github.com/relaton/relaton-data-ecma[ecma-standards] prefetched dataset as a data source. The dataset contains documents from ECMA https://www.ecma-international.org/publications-and-standards/standards/[Standards], https://www.ecma-international.org/publications-and-standards/technical-reports/[Technical Reports], and https://www.ecma-international.org/publications-and-standards/mementos/[Mementos] pages.
+
+The method `RelatonEcma::DataFetcher.new(output: "data", format: "yaml").fetch` fetches all the documents from the pages and saves them to the `./data` folder in YAML format.
+Arguments:
+
+- `output` - folder to save documents (default './data').
+- `format` - the format in which the documents are saved. Possible formats are: `yaml`, `xml`, `bibxxml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonEcma::DataFetcher.new.fetch
+Started at: 2022-06-23 09:36:55 +0200
+Stopped at: 2022-06-23 09:36:58 +0200
+Done in: 752 sec.
+=> nil
+----
+
 == Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/grammars/basicdoc.rng

@@ -522,7 +522,6 @@
       <value>tip</value>
       <value>important</value>
       <value>caution</value>
-      <value>statement</value>
     </choice>
   </define>
   <define name="figure">

data/grammars/biblio.rng

@@ -216,6 +216,9 @@
       <optional>
         <ref name="fullname"/>
       </optional>
+      <zeroOrMore>
+        <ref name="credential"/>
+      </zeroOrMore>
       <zeroOrMore>
         <ref name="affiliation"/>
       </zeroOrMore>
@@ -232,6 +235,11 @@
       <ref name="FullNameType"/>
     </element>
   </define>
+  <define name="credential">
+    <element name="credential">
+      <text/>
+    </element>
+  </define>
   <define name="FullNameType">
     <choice>
       <group>
@@ -305,7 +313,9 @@
       <zeroOrMore>
         <ref name="affiliationdescription"/>
       </zeroOrMore>
-      <ref name="organization"/>
+      <optional>
+        <ref name="organization"/>
+      </optional>
     </element>
   </define>
   <define name="affiliationname">
@@ -1316,7 +1326,7 @@
       <value>commentaryOf</value>
       <value>hasCommentary</value>
       <value>related</value>
-      <value>complements</value>
+      <value>hasComplement</value>
       <value>complementOf</value>
       <value>obsoletes</value>
       <value>obsoletedBy</value>

data/lib/relaton_ecma/data_fetcher.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "English"
+require "mechanize"
+require "relaton_ecma"
+
+module RelatonEcma
+  class DataFetcher
+    URL = "https://www.ecma-international.org/publications-and-standards/"
+
+    # @param [String] :output directory to output documents
+    # @param [String] :format output format (xml, yaml, bibxml)
+    def initialize(output: "data", format: "yaml")
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+      @files = []
+      @index = Relaton::Index.find_or_create :ECMA
+      @agent = Mechanize.new
+      @agent.user_agent_alias = Mechanize::AGENT_ALIASES.keys[rand(21)]
+    end
+
+    # @param bib [RelatonItu::ItuBibliographicItem]
+    def write_file(bib) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      id = bib.docidentifier[0].id.gsub(%r{[/\s]}, "_")
+      id += "-#{bib.edition.content.gsub('.', '_')}" if bib.edition
+      extent = bib.extent.detect { |e| e.type == "volume" }
+      id += "-#{extent.reference_from}" if extent
+      file = "#{@output}/#{id}.#{@ext}"
+      if @files.include? file
+        warn "Duplicate file #{file}"
+      else
+        @files << file
+        File.write file, render_doc(bib), encoding: "UTF-8"
+        @index.add_or_update index_id(bib), file
+      end
+    end
+
+    def index_id(bib)
+      { id: bib.docidentifier[0].id }.tap do |i|
+        i[:ed] = bib.edition.content if bib.edition
+        extent = bib.extent.detect { |e| e.type == "volume" }
+        i[:vol] = extent.reference_from if extent
+      end
+    end
+
+    def render_doc(bib)
+      case @format
+      when "yaml" then bib.to_hash.to_yaml
+      when "xml" then bib.to_xml bibdata: true
+      when "bibxml" then bib.to_bibxml
+      end
+    end
+
+    # @param hit [Nokogiri::XML::Element]
+    def parse_page(hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      DataParser.new(hit).parse.each { |item| write_file item }
+    end
+
+    # @param type [String]
+    def html_index(type) # rubocop:disable Metrics/MethodLength
+      result = @agent.get "#{URL}#{type}/"
+      # @last_call_time = Time.now
+      result.xpath(
+        "//li/span[1]/a",
+        "//div[contains(@class, 'entry-content-wrapper')][.//a[.='Download']]",
+      ).each do |hit|
+        # workers << hit
+        parse_page(hit)
+      rescue StandardError => e
+        warn e.message
+        warn e.backtrace
+      end
+    end
+
+    #
+    # Fetch data from Ecma website.
+    #
+    # @return [void]
+    #
+    def fetch
+      t1 = Time.now
+      puts "Started at: #{t1}"
+
+      FileUtils.mkdir_p @output
+
+      html_index "standards"
+      html_index "technical-reports"
+      html_index "mementos"
+      @index.save
+
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+  end
+end

data/lib/relaton_ecma/data_parser.rb

@@ -0,0 +1,215 @@
+module RelatonEcma
+  class DataParser
+    MATTRS = %i[docid title date link].freeze
+    ATTRS = MATTRS + %i[abstract relation edition].freeze
+
+    #
+    # Initialize parser
+    #
+    # @param [Nokogiri::XML::Element] hit document hit
+    #
+    def initialize(hit)
+      @hit = hit
+      @bib = {
+        type: "standard", language: ["en"], script: ["Latn"], place: ["Geneva"], doctype: "document"
+      }
+      @agent = Mechanize.new
+    end
+
+    def parse # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      if @hit[:href]
+        @agent.user_agent_alias = Mechanize::AGENT_ALIASES.keys[rand(21)]
+        @doc = get_page @hit[:href]
+        ATTRS.each { |a| @bib[a] = send "fetch_#{a}" }
+      else
+        MATTRS.each { |a| @bib[a] = send "fetch_mem_#{a}" }
+      end
+      @bib[:contributor] = contributor
+      items = [BibliographicItem.new(**@bib)]
+      items + parse_editions
+    end
+
+    #
+    # Get page with retries
+    #
+    # @param [String] url url to fetch
+    #
+    # @return [Mechanize::Page] document
+    #
+    def get_page(url)
+      3.times do |n|
+        sleep n
+        doc = @agent.get url
+        return doc
+      rescue StandardError => e
+        warn e.message
+      end
+    end
+
+    #
+    # Parse editions
+    #
+    # @param [Mechanize::Page] doc document
+    # @param [Hash] bib bibliographic item the last edition
+    #
+    # @return [void]
+    #
+    def parse_editions # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+      return [] unless @doc
+
+      docid = @bib[:docid]
+      @doc.xpath('//div[@id="main"]/div[1]/div/main/article/div/div/standard/div/ul/li').map do |hit|
+        id, ed, @bib[:date], vol = edition_id_parts hit.at("./span", "./a").text
+        @bib[:link] = edition_link(hit) + edition_translation_link(ed)
+        next if ed.nil? || ed.empty?
+
+        @bib[:docid] = id.nil? || id.empty? ? docid : fetch_docid(id)
+        @bib[:edition] = RelatonBib::Edition.new(content: ed)
+        @bib[:extent] = vol && [RelatonBib::Locality.new("volume", vol)]
+        BibliographicItem.new(**@bib)
+      end.compact
+    end
+
+    def edition_link(hit)
+      { "src" => hit.at("./a"), "pdf" => hit.at("./span/a") }.map do |type, a|
+        RelatonBib::TypedUri.new(type: type, content: a[:href]) if a
+      end.compact
+    end
+
+    #
+    # Parse edition and date
+    #
+    # @param [String] text identifier text
+    #
+    # @return [Array<String,nil,Array<RelatonBib::BibliographicDate>>] edition and date
+    #
+    def edition_id_parts(text) # rubocop:disable Metrics/MethodLength
+      %r{^
+        (?<id>\w+(?:[\d-]+|\sTR/\d+)),?\s
+        (?:Volume\s(?<vol>[\d.]+),?\s)?
+        (?<ed>[\d.]+)(?:st|nd|rd|th)?\sedition
+        (?:[,.]\s(?<dt>\w+\s\d+))?
+      }x =~ text
+      date = [dt].compact.map do |d|
+        on = Date.strptime(d, "%B %Y").strftime("%Y-%m")
+        RelatonBib::BibliographicDate.new(type: "published", on: on)
+      end
+      [id, ed, date, vol]
+    end
+
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    def fetch_docid(id = nil)
+      id ||= @hit.text
+      [RelatonBib::DocumentIdentifier.new(type: "ECMA", id: id, primary: true)]
+    end
+
+    # @return [Array<RelatonBib::TypedUri>]
+    def fetch_link # rubocop:disable Metrics/AbcSize
+      link = []
+      link << RelatonBib::TypedUri.new(type: "src", content: @hit[:href]) if @hit[:href]
+      ref = @doc.at('//div[@class="ecma-item-content-wrapper"]/span/a',
+                    '//div[@class="ecma-item-content-wrapper"]/a')
+      link << RelatonBib::TypedUri.new(type: "pdf", content: ref[:href]) if ref
+      link + edition_translation_link(@bib[:edition]&.content)
+    end
+
+    def fetch_mem_link
+      @hit.xpath("./div/section/div/p/a").map do |a|
+        RelatonBib::TypedUri.new(type: "pdf", content: a[:href])
+      end
+    end
+
+    def edition_translation_link(edition)
+      translation_link.select { |l| l[:ed] == edition }.map { |l| l[:link] }
+    end
+
+    def translation_link
+      return [] unless @doc
+
+      @translation_link ||= @doc.xpath("//main/article/div/div/standard/div[2]/ul/li").map do |l|
+        a = l.at("span/a")
+        id = l.at("span").text
+        %r{\w+[\d-]+,\s(?<lang>\w+)\sversion,\s(?<ed>[\d.]+)(?:st|nd|rd|th)\sedition} =~ id
+        case lang
+        when "Japanese"
+          { ed: ed, link: RelatonBib::TypedUri.new(type: "pdf", language: "ja", script: "Jpan", content: a[:href]) }
+        end
+      end.compact
+    end
+
+    # @return [Array<Hash>]
+    def fetch_title
+      @doc.xpath('//p[@class="ecma-item-short-description"]').map do |t|
+        { content: t.text.strip, language: "en", script: "Latn" }
+      end
+    end
+
+    # @return [Array<RelatonBib::FormattedString>]
+    def fetch_abstract
+      content = @doc.xpath('//div[@class="ecma-item-content"]/p').map do |a|
+        a.text.strip.squeeze(" ").gsub(/\r\n/, "")
+      end.join "\n"
+      return [] if content.empty?
+
+      [RelatonBib::FormattedString.new(content: content, language: "en", script: "Latn")]
+    end
+
+    # @return [Array<RelatonBib::BibliographicDate>]
+    def fetch_date
+      @doc.xpath('//p[@class="ecma-item-edition"]').map do |d|
+        date = d.text.split(", ").last
+        RelatonBib::BibliographicDate.new type: "published", on: date
+      end
+    end
+
+    # @return [Array<Hash>]
+    def fetch_relation # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
+      @doc.xpath("//ul[@class='ecma-item-archives']/li").map do |rel|
+        ref, ed, date, vol = edition_id_parts rel.at("span").text
+        next if ed.nil? || ed.empty?
+
+        fref = RelatonBib::FormattedRef.new content: ref, language: "en", script: "Latn"
+        docid = RelatonBib::DocumentIdentifier.new(type: "ECMA", id: ref, primary: true)
+        link = rel.xpath("span/a").map { |l| RelatonBib::TypedUri.new type: "pdf", content: l[:href] }
+        edition = RelatonBib::Edition.new content: ed
+        extent = vol && [RelatonBib::Locality.new("volume", vol)]
+        bibitem = BibliographicItem.new(
+          docid: [docid], formattedref: fref, date: date, edition: edition,
+          link: link, extent: extent
+        )
+        { type: "updates", bibitem: bibitem }
+      end.compact
+    end
+
+    #
+    # @return [RelatonBib::Edition, nil]
+    #
+    def fetch_edition
+      cnt = @doc.at('//p[@class="ecma-item-edition"]')&.text&.match(/^\d+(?=(?:st|nd|th|rd))/)&.to_s
+      RelatonBib::Edition.new(content: cnt) if cnt && !cnt.empty?
+    end
+
+    def contributor
+      org = RelatonBib::Organization.new name: "Ecma International"
+      [{ entity: org, role: [{ type: "publisher" }] }]
+    end
+
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    def fetch_mem_docid
+      code = "ECMA MEM/#{@hit.at('div[1]//p').text}"
+      fetch_docid code
+    end
+
+    def fetch_mem_date
+      date = @hit.at("div[2]//p").text
+      on = Date.strptime(date, "%B %Y").strftime "%Y-%m"
+      [RelatonBib::BibliographicDate.new(type: "published", on: on)]
+    end
+
+    def fetch_mem_title
+      year = @hit.at("div[1]//p").text
+      content = "\"Memento #{year}\" for year #{year}"
+      [{ content: content, language: "en", script: "Latn" }]
+    end
+  end
+end

data/lib/relaton_ecma/ecma_bibliography.rb

@@ -3,11 +3,36 @@
 module RelatonEcma
   # IETF bibliography module
   module EcmaBibliography
+    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-ecma/master/"
+
     class << self
-      # @param code [String] the ECMA standard Code to look up (e..g "ECMA-6")
-      # @return [RelatonBib::BibliographicEcma]
-      def search(code)
-        Scrapper.scrape_page code
+      #
+      # Search for a reference on the IETF website.
+      #
+      # @param ref [String] the ECMA standard reference to look up (e..g "ECMA-6")
+      #
+      # @return [Array<Hash>]
+      #
+      def search(ref)
+        refparts = parse_ref ref
+        return false unless refparts
+
+        index = Relaton::Index.find_or_create :ECMA, url: "#{ENDPOINT}index.zip"
+        index.search { |row| match_ref refparts, row }
+      end
+
+      def parse_ref(ref)
+        %r{^
+          (?<id>ECMA(?:[\d-]+|\s\w+/\d+))
+          (?:\sed(?<ed>[\d.]+))?
+          (?:\svol(?<vol>\d+))?
+        }x.match ref
+      end
+
+      def match_ref(refparts, row)
+        row[:id][:id] == refparts[:id] &&
+          (refparts[:ed].nil? || row[:id][:ed] == refparts[:ed]) &&
+          (refparts[:vol].nil? || row[:id][:vol] == refparts[:vol])
       end
 
       # @param code [String] the ECMA standard Code to look up (e..g "ECMA-6")
@@ -16,15 +41,36 @@ module RelatonEcma
       # @return [RelatonEcma::BibliographicItem] Relaton of reference
       def get(code, _year = nil, _opts = {})
         warn "[relaton-ecma] (\"#{code}\") fetching..."
-        result = search code
+        result = fetch_doc(code)
         if result
           warn "[relaton-ecma] (\"#{code}\") found #{result.docidentifier.first.id}"
+          # item
         else
-          warn "[relaton-ecma] WARNING no match found online for #{code}. "\
+          warn "[relaton-ecma] WARNING no match found online for #{code}. " \
                "The code must be exactly like it is on the standards website."
         end
         result
       end
+
+      def compare_edition_volume(aaa, bbb)
+        comp = bbb[:id][:ed] <=> aaa[:id][:ed]
+        comp.zero? ? aaa[:id][:vol] <=> bbb[:id][:vol] : comp
+      end
+
+      def fetch_doc(code) # rubocop:disable Metrics/AbcSize
+        row = search(code).min { |a, b| compare_edition_volume a, b }
+        return unless row
+
+        url = "#{ENDPOINT}#{row[:file]}"
+        doc = OpenURI.open_uri url
+        hash = YAML.safe_load doc
+        hash["fetched"] = Date.today.to_s
+        BibliographicItem.from_hash hash
+      rescue OpenURI::HTTPError => e
+        return if e.io.status.first == "404"
+
+        raise RelatonBib::RequestError, "No document found for #{code} reference. #{e.message}"
+      end
     end
   end
 end

data/lib/relaton_ecma/processor.rb

@@ -7,6 +7,7 @@ module RelatonEcma
       @prefix = "ECMA"
       @defaultprefix = /^ECMA(-|\s)/
       @idtype = "ECMA"
+      @datasets = %w[ecma-standards]
     end
 
     # @param code [String]
@@ -17,6 +18,18 @@ module RelatonEcma
       ::RelatonEcma::EcmaBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] source source name (iec-harmonized-all, iec-harmonized-latest)
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format output format (xml, yaml, bibxml)
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.new(**opts).fetch
+    end
+
     # @param xml [String]
     # @return [RelatonEcma::BibliographicItem]
     def from_xml(xml)
@@ -34,5 +47,12 @@ module RelatonEcma
     def grammar_hash
       @grammar_hash ||= ::RelatonEcma.grammar_hash
     end
+
+    #
+    # Remove index file
+    #
+    def remove_index_file
+      Relaton::Index.find_or_create(:ECMA, url: true).remove_file
+    end
   end
 end

data/lib/relaton_ecma/version.rb

@@ -1,3 +1,3 @@
 module RelatonEcma
-  VERSION = "1.14.0".freeze
+  VERSION = "1.14.2".freeze
 end

data/lib/relaton_ecma.rb

@@ -1,13 +1,15 @@
 require "nokogiri"
 require "open-uri"
 require "yaml"
+require "relaton/index"
 require "relaton_bib"
 require "relaton_ecma/version"
 require "relaton_ecma/bibliographic_item"
 require "relaton_ecma/xml_parser"
 require "relaton_ecma/hash_converter"
-require "relaton_ecma/scrapper"
 require "relaton_ecma/ecma_bibliography"
+require "relaton_ecma/data_fetcher"
+require "relaton_ecma/data_parser"
 
 module RelatonEcma
   # Returns hash of XML reammar

data/relaton_ecma.gemspec

@@ -27,15 +27,10 @@ Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  # spec.add_development_dependency "debase"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "rake", "~> 10.0"
-  # spec.add_development_dependency "ruby-debug-ide"
-  spec.add_development_dependency "ruby-jing"
-  spec.add_development_dependency "simplecov"
-  spec.add_development_dependency "vcr"
-  spec.add_development_dependency "webmock"
 
+  spec.add_dependency "mechanize", "~> 2.7"
   spec.add_dependency "relaton-bib", "~> 1.14.0"
+  spec.add_dependency "relaton-index", "~> 0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-ecma
 version: !ruby/object:Gem::Version
-  version: 1.14.0
+  version: 1.14.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-12-03 00:00:00.000000000 Z
+date: 2023-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -24,20 +24,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: pry-byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -53,75 +39,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: ruby-jing
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '2.7'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 1.14.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.14.0
 - !ruby/object:Gem::Dependency
-  name: relaton-bib
+  name: relaton-index
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: 0.2.0
 description: "RelatonEcma: retrieve ECMA Standards for bibliographic use \nusing the
   BibliographicItem model.\n"
 email:
@@ -148,10 +106,11 @@ files:
 - grammars/relaton-ecma.rng
 - lib/relaton_ecma.rb
 - lib/relaton_ecma/bibliographic_item.rb
+- lib/relaton_ecma/data_fetcher.rb
+- lib/relaton_ecma/data_parser.rb
 - lib/relaton_ecma/ecma_bibliography.rb
 - lib/relaton_ecma/hash_converter.rb
 - lib/relaton_ecma/processor.rb
-- lib/relaton_ecma/scrapper.rb
 - lib/relaton_ecma/version.rb
 - lib/relaton_ecma/xml_parser.rb
 - relaton_ecma.gemspec
@@ -175,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.4.9
 signing_key:
 specification_version: 4
 summary: 'RelatonIetf: retrieve ECMA Standards for bibliographic use using the BibliographicItem

data/lib/relaton_ecma/scrapper.rb

@@ -1,29 +0,0 @@
-module RelatonEcma
-  module Scrapper
-    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-ecma/master/data/".freeze
-
-    class << self
-      # @param code [String]
-      # @return [RelatonBib::BibliographicItem]
-      def scrape_page(code)
-        url = "#{ENDPOINT}#{code.gsub(/[\/\s]/, '_').upcase}.yaml"
-        parse_page url
-      rescue OpenURI::HTTPError => e
-        return if e.io.status.first == "404"
-
-        raise RelatonBib::RequestError, "No document found for #{code} reference. #{e.message}"
-      end
-
-      private
-
-      # @param url [String]
-      # @retrurn [RelatonEcma::BibliographicItem]
-      def parse_page(url)
-        doc = OpenURI.open_uri url
-        hash = YAML.safe_load(doc)
-        hash["fetched"] = Date.today.to_s
-        BibliographicItem.from_hash hash
-      end
-    end
-  end
-end