relaton-cie 1.13.0 → 1.14.1

data/lib/relaton_cie/bibliographic_item.rb

@@ -0,0 +1,7 @@
+module RelatonCie
+  class BibliographicItem < RelatonBib::BibliographicItem
+    def ext_schema
+      @ext_schema ||= schema_versions["relaton-model-cie"]
+    end
+  end
+end

data/lib/relaton_cie/cie_bibliography.rb

@@ -13,14 +13,14 @@ module RelatonCie
       # @param code [String] the ECMA standard Code to look up (e..g "ECMA-6")
       # @param year [String] not used
       # @param opts [Hash] not used
-      # @return [RelatonBib::BibliographicItem] Relaton of reference
+      # @return [RelatonCie::BibliographicItem] Relaton of reference
       def get(code, _year = nil, _opts = {})
         warn "[relaton-cie] (\"#{code}\") fetching..."
         result = search code
         if result
           warn "[relaton-cie] (\"#{code}\") found #{result.docidentifier.first.id}"
         else
-          warn "[relaton-cie] WARNING no match found online for #{code}. "\
+          warn "[relaton-cie] WARNING no match found online for #{code}. " \
                "The code must be exactly like it is on the standards website."
         end
         result

data/lib/relaton_cie/data_fetcher.rb

@@ -10,35 +10,64 @@ module RelatonCie
     URL = "https://www.techstreet.com/cie/searches/31156444?page=1&per_page=100"
 
     def initialize(output, format)
-      @agent = Mechanize.new
       @output = output
       @format = format
+      @files = []
+      @ext = format == "bibxml" ? "xml" : format
+    end
+
+    def agent
+      @agent ||= Mechanize.new
+    end
+
+    def index
+      @index ||= Relaton::Index.find_or_create :cie, file: "index-v1.yaml"
     end
 
     # @param hit [Nokogiri::HTML::Document]
     # @param doc [Mechanize::Page]
     # @return [Array<RelatonBib::DocumentIdentifier>]
-    def fetch_docid(hit, doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-      code = hit.at("h3/a").text.strip.sub(/\u25b9/, "").gsub(" / ", "/")
+    def fetch_docid(hit, doc)
+      code, code2 = parse_code hit, doc
+      docid = [RelatonBib::DocumentIdentifier.new(type: "CIE", id: code, primary: true)]
+      if code2
+        type2 = code2.match(/\w+/).to_s
+        docid << RelatonBib::DocumentIdentifier.new(type: type2, id: code2.strip)
+      end
+      isbn = doc.at('//dt[contains(.,"ISBN")]/following-sibling::dd')
+      docid << RelatonBib::DocumentIdentifier.new(type: "ISBN", id: isbn.text.strip) if isbn
+      docid
+    end
+
+    def parse_code(hit, doc = nil)
+      code = hit.at("h3/a").text.strip.squeeze(" ").sub(/\u25b9/, "").gsub(" / ", "/")
       c2idx = %r{(?:\(|/)(?<c2>(?:ISO|IEC)\s[^()]+)} =~ code
       code = code[0...c2idx].strip if c2idx
+      [primary_code(code, doc), c2]
+    end
+
+    def primary_code(code, doc = nil)
       /^(?<code1>[^(]+)(?:\((?<code2>\w+\d+,(?:\sPages)?[^)]+))?/ =~ code
-      if code1.match?(/^CIE/)
-        c = code1.size > 25 && code2 ? "CIE #{code2.sub(/,(\sPages)?/, '')}" : code1
-        add = doc.at("//hgroup/h2")&.text&.match(/(Add)endum\s(\d+)$/)
-        c += " #{add[1]} #{add[2]}" if add
-      elsif (pcode = doc.at('//dt[.="Product Code(s):"]/following-sibling::dd'))
-        c = "CIE #{pcode.text.strip.match(/[^,]+/)}"
+      if code1&.match?(/^CIE/)
+        parse_cie_code code1, code2, doc
+      elsif (pcode = doc&.at('//dt[.="Product Code(s):"]/following-sibling::dd'))
+        "CIE #{pcode.text.strip.match(/[^,]+/)}"
       else
-        num = code.match(/(?<=\()\w{2}\d+,.+(?=\))/).to_s.gsub(/,(?=\s)/, "")
-          .gsub(/,(?=\S)/, " ")
-        c = "CIE #{num}"
+        num = code.match(/(?<=\()\w{2}\d+,.+(?=\))/).to_s.gsub(/,(?=\s)/, "").gsub(/,(?=\S)/, " ")
+        "CIE #{num}"
       end
-      docid = [RelatonBib::DocumentIdentifier.new(type: "CIE", id: c, primary: true)]
-      isbn = doc.at('//dt[contains(.,"ISBN")]/following-sibling::dd')
-      docid << RelatonBib::DocumentIdentifier.new(type: c2.match(/\w+/).to_s, id: c2.strip) if c2
-      docid << RelatonBib::DocumentIdentifier.new(type: "ISBN", id: isbn.text.strip) if isbn
-      docid
+    end
+
+    def parse_cie_code(code1, code2, doc = nil) # rubocop:disable Metrics/CyclomaticComplexity
+      code = code1.size > 25 && code2 ? "CIE #{code2.sub(/,(\sPages)?/, '')}" : code1
+      add = doc&.at("//hgroup/h2")&.text&.match(/(Add)endum\s(\d+)$/)
+      return code unless add
+
+      "#{code} #{add[1]} #{add[2]}"
+    end
+
+    def fetch_docnumber(hit)
+      parse_code(hit).first.sub(/\w+\s/, "")
     end
 
     # @param doc [Mechanize::Page]
@@ -63,7 +92,7 @@ module RelatonCie
     # @param doc [Mechanize::Page]
     # @return [String]
     def fetch_edition(doc)
-      doc.at('//dt[.="Edition:"]/following-sibling::dd')&.text&.match(/^\d+(?=th)/)&.to_s
+      doc.at('//dt[.="Edition:"]/following-sibling::dd')&.text&.match(/^\d+(?=(st|nd|rd|th))/)&.to_s
     end
 
     # @param doc [Mechanize::Page]
@@ -78,7 +107,7 @@ module RelatonCie
         on = ref.at("p/time")
         date = [RelatonBib::BibliographicDate.new(type: "published", on: on[:datetime])]
         link = [RelatonBib::TypedUri.new(type: "src", content: url)]
-        bibitem = RelatonBib::BibliographicItem.new docid: docid, title: title, link: link, date: date
+        bibitem = BibliographicItem.new docid: docid, title: title, link: link, date: date
         type = ref.at('//li/i[contains(@class,"historical")]') ? "updates" : "updatedBy"
         { type: type, bibitem: bibitem }
       end
@@ -103,14 +132,14 @@ module RelatonCie
     # @param doc [Mechanize::Page]
     # @return [Array<Hash>]
     def fetch_contributor(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity
-      authors = doc.xpath('//hgroup/p[not(@class="pub_date")]').text
+      authors = doc.xpath('//hgroup/p[not(@class="pub_date")]').text.gsub "\"", ""
       contribs = []
       until authors.empty?
         /^(?<sname1>\S+(?:\sder?\s)?[^\s,]+)
         (?:,?\s(?<sname2>[\w-]{2,})(?=,\s+\w\.))?
         (?:,?\s(?<fname>[\w-]{2,})(?!,\s+\w\.))?
         (?:(?:\s?,\s?|\s)(?<init>(?:\w(?:\s?\.|\s|,|$)[\s-]?)+))?
-        (?:(?:,\s?|\s|\.|(?<=\s))(?:and\s)?)?/x =~ authors
+        (?:(?:,\s*|\s+|\.|(?<=\s))(?:and\s)?)?/x =~ authors
         raise StandardError, "Author name not found in \"#{authors}\"" unless $LAST_MATCH_INFO
 
         authors.sub! $LAST_MATCH_INFO.to_s, ""
@@ -131,25 +160,33 @@ module RelatonCie
       contribs << { entity: org, role: [{ type: "publisher" }] }
     end
 
-    # @param bib [RelatonItu::ItuBibliographicItem]
-    def write_file(bib)
+    # @param bib [RelatonCie::BibliographicItem]
+    def write_file(bib) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       id = bib.docidentifier[0].id.gsub(%r{[/\s\-:.]}, "_")
       file = "#{@output}/#{id.upcase}.#{@format}"
-      # if File.exist? file
-      #   warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
-      #   warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
-      # else
-      out = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
-      File.write file, out, encoding: "UTF-8"
-      # end
+      if @files.include? file
+        warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
+        warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
+      else @files << file
+      end
+      index.add_or_update bib.docidentifier[0].id, file
+      File.write file, content(bib), encoding: "UTF-8"
+    end
+
+    def content(bib)
+      case @format
+      when "xml" then bib.to_xml(bibdata: true)
+      when "yaml" then bib.to_hash.to_yaml
+      when "bibxml" then bib.to_bibxml
+      end
     end
 
     # @param hit [Nokogiri::HTML::Element]
     def parse_page(hit) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       url = "https://www.techstreet.com#{hit.at('h3/a')[:href]}"
-      doc = time_req { @agent.get url }
-      item = RelatonBib::BibliographicItem.new(
-        type: "standard", link: fetch_link(url), fetched: Date.today.to_s,
+      doc = time_req { agent.get url }
+      item = BibliographicItem.new(
+        type: "standard", link: fetch_link(url), docnumber: fetch_docnumber(hit),
         docid: fetch_docid(hit, doc), title: fetch_title(doc),
         abstract: fetch_abstract(doc), date: fetch_date(doc),
         edition: fetch_edition(doc), contributor: fetch_contributor(doc),
@@ -164,10 +201,14 @@ module RelatonCie
     end
 
     def fetch(url)
-      result = time_req { @agent.get url }
+      result = time_req { agent.get url }
       result.xpath("//li[@data-product]").each { |hit| parse_page hit }
       np = result.at '//a[@class="next_page"]'
-      fetch "https://www.techstreet.com#{np[:href]}" if np
+      if np
+        fetch "https://www.techstreet.com#{np[:href]}"
+      else
+        index.save
+      end
     end
 
     def time_req
@@ -182,7 +223,7 @@ module RelatonCie
       t1 = Time.now
       puts "Started at: #{t1}"
 
-      FileUtils.mkdir output unless Dir.exist? output
+      FileUtils.mkdir_p output
       new(output, format).fetch URL
 
       t2 = Time.now

data/lib/relaton_cie/hash_converter.rb

@@ -0,0 +1,11 @@
+module RelatonCie
+  class HashConverter < RelatonBib::HashConverter
+    class << self
+      # @param item_hash [Hash]
+      # @return [RelatonCie::BibliographicItem]
+      def bib_item(item_hash)
+        BibliographicItem.new(**item_hash)
+      end
+    end
+  end
+end

data/lib/relaton_cie/processor.rb

@@ -13,7 +13,7 @@ module RelatonCie
     # @param code [String]
     # @param date [String, NilClass] year
     # @param opts [Hash]
-    # @return [RelatonBib::BibliographicItem]
+    # @return [RelatonCie::BibliographicItem]
     def get(code, date, opts)
       ::RelatonCie::CieBibliography.get(code, date, opts)
     end
@@ -31,21 +31,28 @@ module RelatonCie
     end
 
     # @param xml [String]
-    # @return [RelatonBib::BibliographicItem]
+    # @return [RelatonCie::BibliographicItem]
     def from_xml(xml)
-      ::RelatonBib::XMLParser.from_xml xml
+      ::RelatonCie::XMLParser.from_xml xml
     end
 
     # @param hash [Hash]
-    # @return [RelatonBib::BibliographicItem]
+    # @return [RelatonCie::BibliographicItem]
     def hash_to_bib(hash)
-      ::RelatonBib::BibliographicItem.from_hash hash
+      ::RelatonCie::BibliographicItem.from_hash hash
     end
 
     # Returns hash of XML grammar
     # @return [String]
     def grammar_hash
-      @grammar_hash ||= ::RelatonEcma.grammar_hash
+      @grammar_hash ||= ::RelatonCie.grammar_hash
+    end
+
+    #
+    # Remove index file
+    #
+    def remove_index_file
+      Relaton::Index.find_or_create(:cie, url: true, file: Scrapper::INDEX_FILE).remove_file
     end
   end
 end

data/lib/relaton_cie/scrapper.rb

@@ -1,13 +1,17 @@
 module RelatonCie
   module Scrapper
-    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-cie/master/data/".freeze
+    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-cie/master/".freeze
+    INDEX_FILE = "index-v1.yaml".freeze
 
     class << self
       # @param code [String]
-      # @return [RelatonBib::BibliographicItem]
+      # @return [RelatonCie::BibliographicItem]
       def scrape_page(code)
-        url = "#{ENDPOINT}#{code.gsub(/[\/\s\-:.]/, '_').upcase}.yaml"
-        parse_page url
+        index = Relaton::Index.find_or_create :cie, url: "#{ENDPOINT}index-v1.zip", file: INDEX_FILE
+        row = index.search(code).min_by { |r| r[:id] }
+        return unless row
+
+        parse_page "#{ENDPOINT}#{row[:file]}"
       rescue OpenURI::HTTPError => e
         return if e.io.status.first == "404"
 
@@ -17,11 +21,12 @@ module RelatonCie
       private
 
       # @param url [String]
-      # @retrurn [RelatonBib::BibliographicItem]
+      # @retrurn [RelatoCie::BibliographicItem]
       def parse_page(url)
         doc = OpenURI.open_uri url
         bib_hash = RelatonBib::HashConverter.hash_to_bib YAML.safe_load(doc)
-        RelatonBib::BibliographicItem.new(**bib_hash)
+        bib_hash[:fetched] = Date.today.to_s
+        RelatonCie::BibliographicItem.new(**bib_hash)
       end
     end
   end

data/lib/relaton_cie/version.rb

@@ -1,3 +1,3 @@
 module RelatonCie
-  VERSION = "1.13.0".freeze
+  VERSION = "1.14.1".freeze
 end

data/lib/relaton_cie/xml_parser.rb

@@ -0,0 +1,13 @@
+module RelatonCie
+  class XMLParser < RelatonBib::XMLParser
+    class << self
+      private
+
+      # @param item_hash [Hash]
+      # @return [RelatonCie::BibliographicItem]
+      def bib_item(item_hash)
+        BibliographicItem.new(**item_hash)
+      end
+    end
+  end
+end

data/lib/relaton_cie.rb

@@ -1,10 +1,16 @@
 require "nokogiri"
 require "open-uri"
+# require "parslet"
+require "relaton/index"
 require "relaton_bib"
+# require "relaton_bib/name_parser"
 require "relaton_cie/version"
+require "relaton_cie/bibliographic_item"
 require "relaton_cie/cie_bibliography"
 require "relaton_cie/scrapper"
 require "relaton_cie/data_fetcher"
+require "relaton_cie/xml_parser"
+require "relaton_cie/hash_converter"
 
 module RelatonCie
   # Returns hash of XML reammar

data/relaton_cie.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   DESCRIPTION
   spec.homepage      = "https://github.com/metanorma/relaton-cie"
   spec.license       = "BSD-2-Clause"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
 
@@ -29,13 +29,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "ruby-jing"
-  spec.add_development_dependency "simplecov"
-  spec.add_development_dependency "vcr"
-  spec.add_development_dependency "webmock"
-
   spec.add_dependency "mechanize", "~> 2.8.0"
-  spec.add_dependency "relaton-bib", "~> 1.13.0"
+  spec.add_dependency "parslet", "~> 2.0.0"
+  spec.add_dependency "relaton-bib", "~> 1.14.0"
+  spec.add_dependency "relaton-index", "~> 0.2.0"
 end

metadata

@@ -1,127 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: relaton-cie
 version: !ruby/object:Gem::Version
-  version: 1.13.0
+  version: 1.14.1
 platform: ruby
 authors:
 - Ribose Inc.
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-08-21 00:00:00.000000000 Z
+date: 2023-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: equivalent-xml
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :development
+        version: 2.8.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: pry-byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: ruby-jing
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.8.0
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: parslet
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 2.0.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: 1.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: 1.14.0
 - !ruby/object:Gem::Dependency
-  name: relaton-bib
+  name: relaton-index
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: 0.2.0
 description: "RelatonEcma: retrieve CIE Standards for bibliographic use \nusing the
   BibliographicItem model.\n"
 email:
@@ -131,6 +75,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/workflows/rake.yml"
+- ".github/workflows/release.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
@@ -142,22 +87,26 @@ files:
 - bin/rspec
 - bin/setup
 - grammars/basicdoc.rng
+- grammars/biblio-standoc.rng
 - grammars/biblio.rng
-- grammars/isodoc.rng
-- grammars/reqt.rng
+- grammars/relaton-cie-compile.rng
+- grammars/relaton-cie.rng
 - lib/relaton_cie.rb
+- lib/relaton_cie/bibliographic_item.rb
 - lib/relaton_cie/cie_bibliography.rb
 - lib/relaton_cie/data_fetcher.rb
+- lib/relaton_cie/hash_converter.rb
 - lib/relaton_cie/processor.rb
 - lib/relaton_cie/scrapper.rb
 - lib/relaton_cie/version.rb
+- lib/relaton_cie/xml_parser.rb
 - relaton_cie.gemspec
 homepage: https://github.com/metanorma/relaton-cie
 licenses:
 - BSD-2-Clause
 metadata:
   homepage_uri: https://github.com/metanorma/relaton-cie
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -165,15 +114,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
-signing_key:
+rubygems_version: 3.3.26
+signing_key: 
 specification_version: 4
 summary: 'RelatonEcma: retrieve CIE Standards for bibliographic use  using the BibliographicItem
   model.'