relaton-cie 1.14.0 → 1.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0de5250a785ffc6f5b0c4e93a1011007bc9ddf72339746c94570ebf7b6a25a78
-  data.tar.gz: 4173a47ac59c47492141c40731fcb498451675ef97430a563686cadcb1388f2c
+  metadata.gz: 5734bdd3bda68a01b807d6852a31d7baca1c5e881beb0920586dcbf5f8a86304
+  data.tar.gz: adae1acf15433e788fc25eb4d1aa96a02acecdf83fae8eff884e26ade4d54363
 SHA512:
-  metadata.gz: f7f5f59752a190ca8769df82b29de6051e6780b59eb81452cc3d2a441f66d6174906233a35085b5845f3eddeb448b876f5057dfe337e5e34a03922bbfbe2c35a
-  data.tar.gz: f7d83a7aefcdbc7bae57b4d31fccb89f82a79285ab746494a5633241263d428ff1abe9be75042361e4ff58f3359578004d1da8c1f4a919b45de9d983b266f2f1
+  metadata.gz: 673fa336c8c794796b9c0b12619b3c16ff2ec0440e428a685892209e8d517d7abac4bf9f5ae838ca2d77c31fe0c711edbcd43f9be861175b72580875e62769d5
+  data.tar.gz: '08387ec9150fd753c355c4fa9a5abab994301698b56bf2a12c1918511cad9fbdc991f1f4c72f407bf996fd978608f09fb5ee6c1fac2c408a8ef953b6acac8e84'

data/.github/workflows/rake.yml

@@ -5,6 +5,7 @@ name: rake
 on:
   push:
     branches: [ master, main ]
+    tags: [ v* ]
   pull_request:
 
 jobs:

data/.github/workflows/release.yml

@@ -7,11 +7,13 @@ on:
     inputs:
       next_version:
         description: |
-          Next release version. Possible values: x.y.z, major, minor, patch or pre|rc|etc
+          Next release version. Possible values: x.y.z, major, minor, patch (or pre|rc|etc).
+          Also, you can pass 'skip' to skip 'git tag' and do 'gem push' for the current version
         required: true
         default: 'skip'
-  push:
-    tags: [ v* ]
+  repository_dispatch:
+    types: [ do-release ]
+
 
 jobs:
   release:

data/.gitignore

@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/vendor/
 
 # rspec failure tracking
 .rspec_status

data/.rubocop.yml

@@ -7,6 +7,6 @@ require: rubocop-rails
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 2.7
 Rails:
   Enabled: false

data/Gemfile

@@ -3,5 +3,11 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in relaton_cie.gemspec
 gemspec
 
+gem "equivalent-xml", "~> 0.6"
+gem "pry-byebug"
 gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
+gem "ruby-jing"
+gem "simplecov"
+gem "vcr"
+gem "webmock"

data/README.adoc

@@ -99,13 +99,13 @@ item = RelatonCie::XMLParser.from_xml File.read("spec/fixtures/bibdata.xml")
 
 === Fetch data
 
-This gem uses the https://www.techstreet.com/cie/searches/31156444 dataset as one of data sources.
+This gem uses the https://www.techstreet.com/cie/searches/31156444 dataset as one of the data sources.
 
 The method `RelatonCie::DataFetcher.fetch(output: "data", format: "yaml")` fetches all the documents from the dataset and saves them to the `./data` folder in YAML format.
 Arguments:
 
 - `output` - folder to save documents (default './data').
-- `format` - the format in which the documents are saved. Possible formats are: `yaml`, `xml` (default `yaml`).
+- `format` - the format in which the documents are saved. Possible formats are: `yaml`, `xml`, `bibxml` (default `yaml`).
 
 [source,ruby]
 ----

data/grammars/basicdoc.rng

@@ -522,7 +522,6 @@
       <value>tip</value>
       <value>important</value>
       <value>caution</value>
-      <value>statement</value>
     </choice>
   </define>
   <define name="figure">

data/grammars/biblio.rng

@@ -216,6 +216,9 @@
       <optional>
         <ref name="fullname"/>
       </optional>
+      <zeroOrMore>
+        <ref name="credential"/>
+      </zeroOrMore>
       <zeroOrMore>
         <ref name="affiliation"/>
       </zeroOrMore>
@@ -232,6 +235,11 @@
       <ref name="FullNameType"/>
     </element>
   </define>
+  <define name="credential">
+    <element name="credential">
+      <text/>
+    </element>
+  </define>
   <define name="FullNameType">
     <choice>
       <group>
@@ -305,7 +313,9 @@
       <zeroOrMore>
         <ref name="affiliationdescription"/>
       </zeroOrMore>
-      <ref name="organization"/>
+      <optional>
+        <ref name="organization"/>
+      </optional>
     </element>
   </define>
   <define name="affiliationname">
@@ -1273,7 +1283,7 @@
       <value>mergedInto</value>
       <value>splits</value>
       <value>splitInto</value>
-      <value>instance</value>
+      <value>instanceOf</value>
       <value>hasInstance</value>
       <value>exemplarOf</value>
       <value>hasExemplar</value>
@@ -1316,7 +1326,7 @@
       <value>commentaryOf</value>
       <value>hasCommentary</value>
       <value>related</value>
-      <value>complements</value>
+      <value>hasComplement</value>
       <value>complementOf</value>
       <value>obsoletes</value>
       <value>obsoletedBy</value>

data/lib/relaton_cie/data_fetcher.rb

@@ -10,35 +10,64 @@ module RelatonCie
     URL = "https://www.techstreet.com/cie/searches/31156444?page=1&per_page=100"
 
     def initialize(output, format)
-      @agent = Mechanize.new
       @output = output
       @format = format
+      @files = []
+      @ext = format == "bibxml" ? "xml" : format
+    end
+
+    def agent
+      @agent ||= Mechanize.new
+    end
+
+    def index
+      @index ||= Relaton::Index.find_or_create :cie, file: "index-v1.yaml"
     end
 
     # @param hit [Nokogiri::HTML::Document]
     # @param doc [Mechanize::Page]
     # @return [Array<RelatonBib::DocumentIdentifier>]
-    def fetch_docid(hit, doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-      code = hit.at("h3/a").text.strip.sub(/\u25b9/, "").gsub(" / ", "/")
+    def fetch_docid(hit, doc)
+      code, code2 = parse_code hit, doc
+      docid = [RelatonBib::DocumentIdentifier.new(type: "CIE", id: code, primary: true)]
+      if code2
+        type2 = code2.match(/\w+/).to_s
+        docid << RelatonBib::DocumentIdentifier.new(type: type2, id: code2.strip)
+      end
+      isbn = doc.at('//dt[contains(.,"ISBN")]/following-sibling::dd')
+      docid << RelatonBib::DocumentIdentifier.new(type: "ISBN", id: isbn.text.strip) if isbn
+      docid
+    end
+
+    def parse_code(hit, doc = nil)
+      code = hit.at("h3/a").text.strip.squeeze(" ").sub(/\u25b9/, "").gsub(" / ", "/")
       c2idx = %r{(?:\(|/)(?<c2>(?:ISO|IEC)\s[^()]+)} =~ code
       code = code[0...c2idx].strip if c2idx
+      [primary_code(code, doc), c2]
+    end
+
+    def primary_code(code, doc = nil)
       /^(?<code1>[^(]+)(?:\((?<code2>\w+\d+,(?:\sPages)?[^)]+))?/ =~ code
-      if code1.match?(/^CIE/)
-        c = code1.size > 25 && code2 ? "CIE #{code2.sub(/,(\sPages)?/, '')}" : code1
-        add = doc.at("//hgroup/h2")&.text&.match(/(Add)endum\s(\d+)$/)
-        c += " #{add[1]} #{add[2]}" if add
-      elsif (pcode = doc.at('//dt[.="Product Code(s):"]/following-sibling::dd'))
-        c = "CIE #{pcode.text.strip.match(/[^,]+/)}"
+      if code1&.match?(/^CIE/)
+        parse_cie_code code1, code2, doc
+      elsif (pcode = doc&.at('//dt[.="Product Code(s):"]/following-sibling::dd'))
+        "CIE #{pcode.text.strip.match(/[^,]+/)}"
       else
-        num = code.match(/(?<=\()\w{2}\d+,.+(?=\))/).to_s.gsub(/,(?=\s)/, "")
-          .gsub(/,(?=\S)/, " ")
-        c = "CIE #{num}"
+        num = code.match(/(?<=\()\w{2}\d+,.+(?=\))/).to_s.gsub(/,(?=\s)/, "").gsub(/,(?=\S)/, " ")
+        "CIE #{num}"
       end
-      docid = [RelatonBib::DocumentIdentifier.new(type: "CIE", id: c, primary: true)]
-      isbn = doc.at('//dt[contains(.,"ISBN")]/following-sibling::dd')
-      docid << RelatonBib::DocumentIdentifier.new(type: c2.match(/\w+/).to_s, id: c2.strip) if c2
-      docid << RelatonBib::DocumentIdentifier.new(type: "ISBN", id: isbn.text.strip) if isbn
-      docid
+    end
+
+    def parse_cie_code(code1, code2, doc = nil) # rubocop:disable Metrics/CyclomaticComplexity
+      code = code1.size > 25 && code2 ? "CIE #{code2.sub(/,(\sPages)?/, '')}" : code1
+      add = doc&.at("//hgroup/h2")&.text&.match(/(Add)endum\s(\d+)$/)
+      return code unless add
+
+      "#{code} #{add[1]} #{add[2]}"
+    end
+
+    def fetch_docnumber(hit)
+      parse_code(hit).first.sub(/\w+\s/, "")
     end
 
     # @param doc [Mechanize::Page]
@@ -63,7 +92,7 @@ module RelatonCie
     # @param doc [Mechanize::Page]
     # @return [String]
     def fetch_edition(doc)
-      doc.at('//dt[.="Edition:"]/following-sibling::dd')&.text&.match(/^\d+(?=th)/)&.to_s
+      doc.at('//dt[.="Edition:"]/following-sibling::dd')&.text&.match(/^\d+(?=(st|nd|rd|th))/)&.to_s
     end
 
     # @param doc [Mechanize::Page]
@@ -132,24 +161,32 @@ module RelatonCie
     end
 
     # @param bib [RelatonCie::BibliographicItem]
-    def write_file(bib)
+    def write_file(bib) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       id = bib.docidentifier[0].id.gsub(%r{[/\s\-:.]}, "_")
       file = "#{@output}/#{id.upcase}.#{@format}"
-      # if File.exist? file
-      #   warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
-      #   warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
-      # else
-      out = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
-      File.write file, out, encoding: "UTF-8"
-      # end
+      if @files.include? file
+        warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
+        warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
+      else @files << file
+      end
+      index.add_or_update bib.docidentifier[0].id, file
+      File.write file, content(bib), encoding: "UTF-8"
+    end
+
+    def content(bib)
+      case @format
+      when "xml" then bib.to_xml(bibdata: true)
+      when "yaml" then bib.to_hash.to_yaml
+      when "bibxml" then bib.to_bibxml
+      end
     end
 
     # @param hit [Nokogiri::HTML::Element]
     def parse_page(hit) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       url = "https://www.techstreet.com#{hit.at('h3/a')[:href]}"
-      doc = time_req { @agent.get url }
+      doc = time_req { agent.get url }
       item = BibliographicItem.new(
-        type: "standard", link: fetch_link(url),
+        type: "standard", link: fetch_link(url), docnumber: fetch_docnumber(hit),
         docid: fetch_docid(hit, doc), title: fetch_title(doc),
         abstract: fetch_abstract(doc), date: fetch_date(doc),
         edition: fetch_edition(doc), contributor: fetch_contributor(doc),
@@ -164,10 +201,14 @@ module RelatonCie
     end
 
     def fetch(url)
-      result = time_req { @agent.get url }
+      result = time_req { agent.get url }
       result.xpath("//li[@data-product]").each { |hit| parse_page hit }
       np = result.at '//a[@class="next_page"]'
-      fetch "https://www.techstreet.com#{np[:href]}" if np
+      if np
+        fetch "https://www.techstreet.com#{np[:href]}"
+      else
+        index.save
+      end
     end
 
     def time_req
@@ -182,7 +223,7 @@ module RelatonCie
       t1 = Time.now
       puts "Started at: #{t1}"
 
-      FileUtils.mkdir output
+      FileUtils.mkdir_p output
       new(output, format).fetch URL
 
       t2 = Time.now

data/lib/relaton_cie/processor.rb

@@ -47,5 +47,12 @@ module RelatonCie
     def grammar_hash
       @grammar_hash ||= ::RelatonCie.grammar_hash
     end
+
+    #
+    # Remove index file
+    #
+    def remove_index_file
+      Relaton::Index.find_or_create(:cie, url: true, file: Scrapper::INDEX_FILE).remove_file
+    end
   end
 end

data/lib/relaton_cie/scrapper.rb

@@ -1,13 +1,17 @@
 module RelatonCie
   module Scrapper
-    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-cie/master/data/".freeze
+    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-cie/master/".freeze
+    INDEX_FILE = "index-v1.yaml".freeze
 
     class << self
       # @param code [String]
       # @return [RelatonCie::BibliographicItem]
       def scrape_page(code)
-        url = "#{ENDPOINT}#{code.gsub(/[\/\s\-:.]/, '_').upcase}.yaml"
-        parse_page url
+        index = Relaton::Index.find_or_create :cie, url: "#{ENDPOINT}index-v1.zip", file: INDEX_FILE
+        row = index.search(code).min_by { |r| r[:id] }
+        return unless row
+
+        parse_page "#{ENDPOINT}#{row[:file]}"
       rescue OpenURI::HTTPError => e
         return if e.io.status.first == "404"
 

data/lib/relaton_cie/version.rb

@@ -1,3 +1,3 @@
 module RelatonCie
-  VERSION = "1.14.0".freeze
+  VERSION = "1.16.0".freeze
 end

data/lib/relaton_cie.rb

@@ -1,6 +1,9 @@
 require "nokogiri"
 require "open-uri"
+# require "parslet"
+require "relaton/index"
 require "relaton_bib"
+# require "relaton_bib/name_parser"
 require "relaton_cie/version"
 require "relaton_cie/bibliographic_item"
 require "relaton_cie/cie_bibliography"

data/relaton_cie.gemspec

@@ -7,16 +7,16 @@ Gem::Specification.new do |spec|
   spec.email         = ["open.source@ribose.com"]
 
   spec.summary       = <<~DESCRIPTION
-    RelatonEcma: retrieve CIE Standards for bibliographic use 
+    RelatonEcma: retrieve CIE Standards for bibliographic use
     using the BibliographicItem model.
   DESCRIPTION
   spec.description = <<~DESCRIPTION
-    RelatonEcma: retrieve CIE Standards for bibliographic use 
+    RelatonEcma: retrieve CIE Standards for bibliographic use
     using the BibliographicItem model.
   DESCRIPTION
   spec.homepage      = "https://github.com/metanorma/relaton-cie"
   spec.license       = "BSD-2-Clause"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
 
@@ -29,13 +29,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "ruby-jing"
-  spec.add_development_dependency "simplecov"
-  spec.add_development_dependency "vcr"
-  spec.add_development_dependency "webmock"
-
   spec.add_dependency "mechanize", "~> 2.8.0"
-  spec.add_dependency "relaton-bib", "~> 1.14.0"
+  spec.add_dependency "parslet", "~> 2.0.0"
+  spec.add_dependency "relaton-bib", "~> 1.16.0"
+  spec.add_dependency "relaton-index", "~> 0.2.0"
 end

metadata

@@ -1,129 +1,74 @@
 --- !ruby/object:Gem::Specification
 name: relaton-cie
 version: !ruby/object:Gem::Version
-  version: 1.14.0
+  version: 1.16.0
 platform: ruby
 authors:
 - Ribose Inc.
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-12-03 00:00:00.000000000 Z
+date: 2023-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: equivalent-xml
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :development
+        version: 2.8.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: pry-byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: ruby-jing
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.8.0
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: parslet
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 2.0.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: 1.16.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: 1.16.0
 - !ruby/object:Gem::Dependency
-  name: relaton-bib
+  name: relaton-index
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
-description: "RelatonEcma: retrieve CIE Standards for bibliographic use \nusing the
-  BibliographicItem model.\n"
+        version: 0.2.0
+description: |
+  RelatonEcma: retrieve CIE Standards for bibliographic use
+  using the BibliographicItem model.
 email:
 - open.source@ribose.com
 executables: []
@@ -162,7 +107,7 @@ licenses:
 - BSD-2-Clause
 metadata:
   homepage_uri: https://github.com/metanorma/relaton-cie
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -170,16 +115,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
-signing_key:
+rubygems_version: 3.3.26
+signing_key: 
 specification_version: 4
-summary: 'RelatonEcma: retrieve CIE Standards for bibliographic use  using the BibliographicItem
+summary: 'RelatonEcma: retrieve CIE Standards for bibliographic use using the BibliographicItem
   model.'
 test_files: []