relaton-iso 1.8.0 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 61646e164a8a384cb744e3e50dafdea336f74cbf7418b59d1b5a30e8aabeea62
-  data.tar.gz: f3c861c4fd11e792f9bd7d7edd0e83d95a5972c6e1590825ce0562360cc5f0d6
+  metadata.gz: c03256162a043a8d8842d3cd31d95dad25487cdde4992b34d44951c650421c3d
+  data.tar.gz: '0295b78c2fbfaaa848e07bb46fabfed6a4ea44c0af0f5e865bf1b25333a4e376'
 SHA512:
-  metadata.gz: 073cf929ab4f17e00651c03443b4bc0d21526994ebd5532a5914f841066f4ec6522d58344ddabf09e69f409243c59c6c721db2a3c1c38c9784b6211cd48d9982
-  data.tar.gz: 2e710926a48e59aba377f89cea8589f4dbb0abe0ddd53a906d9463faf7adf70681f8ef8d8b68f30d8398c9db4b9859a898b243ec53f5f2010dd7bd5a8e99b086
+  metadata.gz: f34e505b97d03f6e8ed481ee87036e51d8748e81e9ab77ea78fda8c7764eca21942a033a06c5e363b0ffacee648e3125cf93a1254897ab60512877836177ce3c
+  data.tar.gz: 9266cb080c0e888b9f6ffaa151d5bc118a6f3b73655cf3f9c8f9d1727dd98e779eef2047f977d492f54c668b58d5f23663374bf2adb8fe306b97f06558b3e973

data/.github/workflows/rake.yml

@@ -16,19 +16,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
+        ruby: [ '3.0', '2.7', '2.6', '2.5' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
-        include:
-          - ruby: '3.0'
-            os: 'ubuntu-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'windows-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'macos-latest'
-            experimental: true
     steps:
       - uses: actions/checkout@v2
         with:

data/.rubocop.yml

@@ -5,6 +5,6 @@
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5
 Rails:
   Enabled: false

data/README.adoc

@@ -31,7 +31,7 @@ Or install it yourself as:
 
 == Usage
 
-=== Search for a standard using keywords
+=== Search for standards using keywords
 
 [source,ruby]
 ----
@@ -50,6 +50,7 @@ item = hit_collection[2].fetch
   [#<RelatonBib::FormattedString:0x007fa5dca88458
     @content=
      "ISO/TS 19115-3:2016 defines an integrated XML implementation of ISO 19115‑1, ..."
+...
 
 item.docidentifier
 => [#<RelatonBib::DocumentIdentifier:0x007fd9ce9c6878 @id="ISO/TS 19115-3:2016", @scope=nil, @type="ISO">,
@@ -59,6 +60,88 @@ item.docidentifier
 => "urn:iso:std:iso-ts:ts:19115:-3:stage-90.92:ed-1:en,fr"
 ----
 
+=== Fetch document by reference and year
+
+[source,ruby]
+----
+item = RelatonIso::IsoBibliography.get "ISO 19115:2003"
+[relaton-iso] ("ISO 19115:2003") fetching...
+[relaton-iso] ("ISO 19115:2003") found ISO 19115:2003
+=> #<RelatonIsoBib::IsoBibliographicItem:0x00007f8c83429e30
+...
+
+item = RelatonIso::IsoBibliography.get "ISO 19115", "2003"
+[relaton-iso] ("ISO 19115") fetching...
+[relaton-iso] ("ISO 19115") found ISO 19115:2003
+=> #<RelatonIsoBib::IsoBibliographicItem:0x00007f8c828d3180
+...
+
+item.docidentifier[0].id
+=> "ISO 19115:2003"
+----
+
+=== Fetch non-part document
+
+[source,ruby]
+----
+item = RelatonIso::IsoBibliography.get "ISO 19115"
+[relaton-iso] ("ISO 19115") fetching...
+[relaton-iso] ("ISO 19115") found ISO 19115:2003
+=> #<RelatonIsoBib::IsoBibliographicItem:0x00007f8c830275a8
+...
+
+item.docidentifier[0].id
+=> "ISO 19115:2003"
+----
+
+=== Fetch part document
+
+[source,ruby]
+----
+item = RelatonIso::IsoBibliography.get "ISO 19115-1"
+[relaton-iso] ("ISO 19115-1") fetching...
+[relaton-iso] ("ISO 19115-1") found ISO 19115-1:2014
+=> #<RelatonIsoBib::IsoBibliographicItem:0x00007f8c83408af0
+...
+
+item.docidentifier[0].id
+=> "ISO 19115-1:2014"
+----
+
+=== Fetch all-parts document
+
+[source,ruby]
+----
+item = RelatonIso::IsoBibliography.get "ISO 19115 (all parts)"
+[relaton-iso] ("ISO 19115") fetching...
+[relaton-iso] ("ISO 19115") found ISO 19115 (all parts)
+=> #<RelatonIsoBib::IsoBibliographicItem:0x00007f8ca216e118
+...
+
+item = RelatonIso::IsoBibliography.get "ISO 19115", nil, all_parts: true
+[relaton-iso] ("ISO 19115") fetching...
+[relaton-iso] ("ISO 19115") found ISO 19115 (all parts)
+=> #<RelatonIsoBib::IsoBibliographicItem:0x00007f8c830f3d38
+...
+
+item.docidentifier[0].id
+=> "ISO 19115 (all parts)"
+
+item = RelatonIso::IsoBibliography.get "ISO 19115-1 (all parts)"
+[relaton-iso] ("ISO 19115-1") fetching...
+[relaton-iso] ("ISO 19115-1") found ISO 19115 (all parts)
+=> #<RelatonIsoBib::IsoBibliographicItem:0x00007f8c8290e5a0
+
+item = RelatonIso::IsoBibliography.get "ISO 19115-1", nil, all_parts: true
+[relaton-iso] ("ISO 19115-1") fetching...
+[relaton-iso] ("ISO 19115-1") found ISO 19115 (all parts)
+=> #<RelatonIsoBib::IsoBibliographicItem:0x00007f8c925355b8
+...
+
+item.docidentifier[0].id
+=> "ISO 19115 (all parts)"
+----
+
 === Search for ISO/IEC Directives
 
 The ISO/IEC Derectives are stored in a static cache in a relaton gem. It needs to use the relaton gem to fetch the ISO/IEC Directives. Folloving reaferences are allowed to fetch:

data/bin/rackup

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rackup' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rack", "rackup")

data/bin/rubocop

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rubocop' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rubocop", "rubocop")

data/bin/ruby-parse

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'ruby-parse' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("parser", "ruby-parse")

data/bin/ruby-rewrite

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'ruby-rewrite' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("parser", "ruby-rewrite")

data/lib/relaton_iso/hit.rb

@@ -4,7 +4,7 @@ module RelatonIso
   # Hit.
   class Hit < RelatonBib::Hit
     # @return [RelatonIsoBib::IsoBibliographicItem]
-    attr_accessor :fetch
+    attr_writer :fetch
 
     # Parse page.
     # @param lang [String, NilClass]
@@ -15,11 +15,12 @@ module RelatonIso
 
     # @return [Integer]
     def sort_weight
-      case hit["publicationStatus"] && hit["publicationStatus"]["key"]
-      when "ENT_ACTIVE" then 0
-      when "ENT_PROGRESS" then 1
-      when "ENT_INACTIVE" then 2
-      else 3
+      case hit[:status] # && hit["publicationStatus"]["key"]
+      when "Published" then 0
+      when "Under development" then 1
+      when "Withdrawn" then 2
+      when "Deleted" then 3
+      else 4
       end
     end
   end

data/lib/relaton_iso/hit_collection.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "algolia"
 require "relaton_iso/hit"
 
 module RelatonIso
@@ -16,18 +17,22 @@ module RelatonIso
     # @param lang [String, NilClass]
     # @return [RelatonIsoBib::IsoBibliographicItem]
     def to_all_parts(lang = nil) # rubocop:disable Metrics/CyclomaticComplexity
-      parts = @array.reject { |h| h.hit["docPart"]&.empty? }
-      hit = parts.min_by { |h| h.hit["docPart"].to_i }
+      # parts = @array.reject { |h| h.hit["docPart"]&.empty? }
+      hit = @array.min_by do |h|
+        IsoBibliography.ref_components(h.hit[:title])[1].to_i
+      end
       return @array.first.fetch lang unless hit
 
       bibitem = hit.fetch lang
       all_parts_item = bibitem.to_all_parts
-      parts.reject { |h| h.hit["docRef"] == hit.hit["docRef"] }.each do |hi|
+      @array.reject { |h| h.hit[:uuid] == hit.hit[:uuid] }.each do |hi|
+        %r{^(?<fr>ISO(?:\s|/)[^-/:()]+(?:-[\w-]+)?(?::\d{4})?
+          (?:/\w+(?:\s\w+)?\s\d+(?:\d{4})?)?)}x =~ hi.hit[:title]
         isobib = RelatonIsoBib::IsoBibliographicItem.new(
-          formattedref: RelatonBib::FormattedRef.new(content: hi.hit["docRef"])
+          formattedref: RelatonBib::FormattedRef.new(content: fr),
         )
         all_parts_item.relation << RelatonBib::DocumentRelation.new(
-          type: "instance", bibitem: isobib
+          type: "instance", bibitem: isobib,
         )
       end
       all_parts_item
@@ -49,8 +54,8 @@ module RelatonIso
 
       hash = YAML.safe_load resp.body
       bib_hash = RelatonIsoBib::HashConverter.hash_to_bib hash
-      bib = RelatonIsoBib::IsoBibliographicItem.new **bib_hash
-      hit = Hit.new({ "docRef" => text }, self)
+      bib = RelatonIsoBib::IsoBibliographicItem.new(**bib_hash)
+      hit = Hit.new({ title: text }, self)
       hit.fetch = bib
       [hit]
     end
@@ -61,21 +66,28 @@ module RelatonIso
     # @return [Array<RelatonIso::Hit>]
     #
     def fetch_iso # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      %r{\s(?<num>\d+)(?:-(?<part>[\d-]+))?} =~ text
-      http = Net::HTTP.new "www.iso.org", 443
-      http.use_ssl = true
-      search = ["status=ENT_ACTIVE,ENT_PROGRESS,ENT_INACTIVE,ENT_DELETED"]
-      search << "docNumber=#{num}"
-      search << "docPartNo=#{part}" if part
-      q = search.join "&"
-      resp = http.get("/cms/render/live/en/sites/isoorg.advancedSearch.do?#{q}",
-                      "Accept" => "application/json, text/plain, */*")
-      return [] if resp.body.empty?
+      # %r{\s(?<num>\d+)(?:-(?<part>[\d-]+))?} =~ text
+      # http = Net::HTTP.new "www.iso.org", 443
+      # http.use_ssl = true
+      # search = ["status=ENT_ACTIVE,ENT_PROGRESS,ENT_INACTIVE,ENT_DELETED"]
+      # search << "docNumber=#{num}"
+      # search << "docPartNo=#{part}" if part
+      # q = search.join "&"
+      # resp = http.get("/cms/render/live/en/sites/isoorg.advancedSearch.do?#{q}",
+      #                 "Accept" => "application/json, text/plain, */*")
+      config = Algolia::Search::Config.new(application_id: "JCL49WV5AR", api_key: "dd1b9e1ab383f4d4817d29cd5e96d3f0")
+      client = Algolia::Search::Client.new config, logger: ::Logger.new($stderr)
+      index = client.init_index "all_en"
+      resp = index.search text, hitsPerPage: 100, filters: "category:standard"
+      # return [] if resp.body.empty?
 
-      json = JSON.parse resp.body
-      json["standards"].map { |h| Hit.new h, self }.sort! do |a, b|
-        if a.sort_weight == b.sort_weight
-          (parse_date(b.hit) - parse_date(a.hit)).to_i
+      # json = JSON.parse resp.body
+      # json["standards"]
+      resp[:hits].map { |h| Hit.new h, self }.sort! do |a, b|
+        if a.sort_weight == b.sort_weight && b.hit[:year] = a.hit[:year]
+          a.hit[:title] <=> b.hit[:title]
+        elsif a.sort_weight == b.sort_weight
+          b.hit[:year] - a.hit[:year]
         else
           a.sort_weight - b.sort_weight
         end
@@ -84,16 +96,16 @@ module RelatonIso
 
     # @param hit [Hash]
     # @return [Date]
-    def parse_date(hit)
-      if hit["publicationDate"]
-        Date.strptime(hit["publicationDate"], "%Y-%m")
-      elsif %r{:(?<year>\d{4})} =~ hit["docRef"]
-        Date.strptime(year, "%Y")
-      elsif hit["newProjectDate"]
-        Date.parse hit["newProjectDate"]
-      else
-        Date.new 0
-      end
-    end
+    # def parse_date(hit)
+    #   if hit["publicationDate"]
+    #     Date.strptime(hit["publicationDate"], "%Y-%m")
+    #   elsif %r{:(?<year>\d{4})} =~ hit["docRef"]
+    #     Date.strptime(year, "%Y")
+    #   elsif hit["newProjectDate"]
+    #     Date.parse hit["newProjectDate"]
+    #   else
+    #     Date.new 0
+    #   end
+    # end
   end
 end

data/lib/relaton_iso/iso_bibliography.rb

@@ -15,8 +15,9 @@ module RelatonIso
         HitCollection.new text.gsub(/\u2013/, "-")
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
              EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
-             Net::ProtocolError, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT
-        raise RelatonBib::RequestError, "Could not access http://www.iso.org"
+             Net::ProtocolError, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT,
+             Algolia::AlgoliaUnreachableHostError => e
+        raise RelatonBib::RequestError, e.message
       end
 
       # @param ref [String] the ISO standard Code to look up (e..g "ISO 9000")
@@ -25,11 +26,15 @@ module RelatonIso
       #   reference is required, :keep_year if undated reference should
       #   return actual reference with year
       # @return [String] Relaton XML serialisation of reference
-      def get(ref, year = nil, opts = {})
+      def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
         code = ref.gsub(/\u2013/, "-")
-        %r{\s(?<num>\d+)(-(?<part>[\d-]+))?(:(?<year1>\d{4}))?} =~ code
+        # %r{\s(?<num>\d+)(?:-(?<part>[\d-]+))?(?::(?<year1>\d{4}))?} =~ code
+        _, _part, year1, = ref_components ref
         year ||= year1
-        opts[:all_parts] ||= !part && opts[:all_parts].nil? # && code2.nil?
+        code.sub! " (all parts)", ""
+        opts[:all_parts] ||= $~ && opts[:all_parts].nil?
+        opts[:keep_year] ||= opts[:keep_year].nil?
+        # code.sub!("#{num}-#{part}", num) if opts[:all_parts] && part
         # if %r[^ISO/IEC DIR].match? code
         #   return RelatonIec::IecBibliography.get(code, year, opts)
         # end
@@ -44,6 +49,16 @@ module RelatonIso
         end
       end
 
+      def ref_components(ref)
+        %r{
+          ^(?<code>ISO(?:\s|/)[^-/:()]+\d+)
+          (?:-(?<part>[\w-]+))?
+          (?::(?<year>\d{4}))?
+          (?:/(?<corr>\w+(?:\s\w+)?\s\d+)(?:(?<coryear>\d{4}))?)?
+        }x =~ ref
+        [code&.strip, part, year, corr, coryear]
+      end
+
       private
 
       # rubocop:disable Metrics/MethodLength
@@ -51,19 +66,19 @@ module RelatonIso
       def fetch_ref_err(code, year, missed_years)
         id = year ? "#{code}:#{year}" : code
         warn "[relaton-iso] WARNING: no match found online for #{id}. "\
-          "The code must be exactly like it is on the standards website."
+             "The code must be exactly like it is on the standards website."
         unless missed_years.empty?
           warn "[relaton-iso] (There was no match for #{year}, though there "\
-            "were matches found for #{missed_years.join(', ')}.)"
+               "were matches found for #{missed_years.join(', ')}.)"
         end
         if /\d-\d/.match? code
           warn "[relaton-iso] The provided document part may not exist, "\
-            "or the document may no longer be published in parts."
+               "or the document may no longer be published in parts."
         else
           warn "[relaton-iso] If you wanted to cite all document parts for "\
-            "the reference, use \"#{code} (all parts)\".\nIf the document is "\
-            "not a standard, use its document type abbreviation (TS, TR, PAS, "\
-            "Guide)."
+               "the reference, use \"#{code} (all parts)\".\nIf the document "\
+               "is not a standard, use its document type abbreviation "\
+               "(TS, TR, PAS, Guide)."
         end
         nil
       end
@@ -76,18 +91,20 @@ module RelatonIso
       # @param opts [Hash]
       # @return [Array<RelatonIso::Hit>]
       def isobib_search_filter(code, opts)
+        ref = remove_part code, opts[:all_parts]
         warn "[relaton-iso] (\"#{code}\") fetching..."
-        result = search(code)
+        result = search(ref)
         res = search_code result, code, opts
         return res unless res.empty?
 
         # try stages
-        if %r{^\w+/[^/]+\s\d+} =~ code # code like ISO/IEC 123, ISO/IEC/IEE 123
+        case code
+        when %r{^\w+/[^/]+\s\d+} # code like ISO/IEC 123, ISO/IEC/IEE 123
           res = try_stages(result, opts) do |st|
             code.sub(%r{^(?<pref>[^\s]+\s)}) { "#{$~[:pref]}#{st} " }
           end
           return res unless res.empty?
-        elsif %r{^\w+\s\d+} =~ code # code like ISO 123
+        when %r{^\w+\s\d+} # code like ISO 123
           res = try_stages(result, opts) do |st|
             code.sub(%r{^(?<pref>\w+)}) { "#{$~[:pref]}/#{st}" }
           end
@@ -103,6 +120,12 @@ module RelatonIso
       end
       # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
+      def remove_part(ref, all_parts)
+        return ref unless all_parts
+
+        ref.sub %r{(\S+\s\d+)[\d-]+}, '\1'
+      end
+
       # @param result [RelatonIso::HitCollection]
       # @param opts [Hash]
       # @return [RelatonIso::HitCollection]
@@ -120,25 +143,15 @@ module RelatonIso
       # @param code [String]
       # @param opts [Hash]
       # @return [RelatonIso::HitCollection]
-      def search_code(result, code, _opts) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity
-        code1, part1, corr1, coryear1 = ref_components code
+      def search_code(result, code, opts) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity
+        code1, part1, _, corr1, coryear1 = ref_components code
         result.select do |i|
-          code2, part2, corr2, coryear2 = ref_components i.hit["docRef"]
-          code1 == code2 && (!part1 || part1 == part2) &&
+          code2, part2, _, corr2, coryear2 = ref_components i.hit[:title]
+          code1 == code2 && (opts[:all_parts] && part2 || !opts[:all_parts] && part1 == part2) &&
             corr1 == corr2 && (!coryear1 || coryear1 == coryear2)
         end
       end
 
-      def ref_components(ref)
-        %r{
-          ^(?<code>ISO(?:\s|/)[^-/:()]+)
-          (?:-(?<part>[^:/]+))?
-          (?::\d{4})?
-          (?:/(?<corr>\w+(?:\s\w+)?\s\d+)(?:(?<coryear>\d{4}))?)?
-        }x =~ ref
-        [code&.strip, part, corr, coryear]
-      end
-
       # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
       # Sort through the results from RelatonIso, fetching them three at a time,
@@ -151,7 +164,7 @@ module RelatonIso
       def isobib_results_filter(result, year, opts)
         missed_years = []
         hits = result.reduce!([]) do |hts, h|
-          if !year || %r{:(?<iyear>\d{4})(?!.*:\d{4})} =~ h.hit["docRef"] && iyear == year
+          if !year || %r{:(?<iyear>\d{4})(?!.*:\d{4})} =~ h.hit[:title] && iyear == year
             hts << h
           else
             missed_years << iyear

data/lib/relaton_iso/scrapper.rb

@@ -7,8 +7,7 @@ require "net/http"
 
 module RelatonIso
   # Scrapper.
-  # rubocop:disable Metrics/ModuleLength
-  module Scrapper
+  module Scrapper # rubocop:disable Metrics/ModuleLength
     DOMAIN = "https://www.iso.org"
 
     TYPES = {
@@ -55,9 +54,9 @@ module RelatonIso
       # @param lang [String, NilClass]
       # @return [Hash]
       def parse_page(hit_data, lang = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        path = "/contents/data/standard#{hit_data['splitPath']}/"\
-        "#{hit_data['csnumber']}.html"
-        doc, url = get_page path
+        # path = "/contents/data/standard#{hit_data['splitPath']}/"\
+        # "#{hit_data['csnumber']}.html"
+        doc, url = get_page "#{hit_data[:path].sub '/sites/isoorg', ''}.html"
 
         # Fetch edition.
         edition = doc&.xpath("//strong[contains(text(), 'Edition')]/..")
@@ -67,24 +66,24 @@ module RelatonIso
 
         RelatonIsoBib::IsoBibliographicItem.new(
           fetched: Date.today.to_s,
-          docid: fetch_docid(hit_data, langs),
+          docid: fetch_docid(doc, edition, langs),
           docnumber: fetch_docnumber(doc),
           edition: edition,
           language: langs.map { |l| l[:lang] },
           script: langs.map { |l| script(l[:lang]) }.uniq,
           title: titles,
-          doctype: fetch_type(hit_data["docRef"]),
+          doctype: fetch_type(hit_data[:title]),
           docstatus: fetch_status(doc),
           ics: fetch_ics(doc),
-          date: fetch_dates(doc, hit_data["docRef"]),
-          contributor: fetch_contributors(hit_data["docRef"]),
+          date: fetch_dates(doc, hit_data[:title]),
+          contributor: fetch_contributors(hit_data[:title]),
           editorialgroup: fetch_workgroup(doc),
           abstract: abstract,
-          copyright: fetch_copyright(hit_data["docRef"], doc),
+          copyright: fetch_copyright(doc),
           link: fetch_link(doc, url),
           relation: fetch_relations(doc),
           place: ["Geneva"],
-          structuredidentifier: fetch_structuredidentifier(doc)
+          structuredidentifier: fetch_structuredidentifier(doc),
         )
       end
 
@@ -94,8 +93,7 @@ module RelatonIso
       # @param doc [Nokigiri::HTML::Document]
       # @param lang [String, NilClass]
       # @return [Array<Array>]
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      def fetch_titles_abstract(doc, lang)
+      def fetch_titles_abstract(doc, lang) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
         titles   = RelatonBib::TypedTitleStringCollection.new
         abstract = []
         langs = languages(doc, lang).reduce([]) do |s, l|
@@ -107,7 +105,11 @@ module RelatonIso
             titles += fetch_title(d, l[:lang])
 
             # Fetch abstracts.
-            abstract_content = d.css("div[itemprop='description'] p").text
+            abstract_content = d.xpath(
+              "//div[@itemprop='description']/p|//div[@itemprop='description']/ul/li",
+            ).map do |a|
+              a.name == "li" ? "- #{a.text}" : a.text
+            end.reject(&:empty?).join("\n")
             unless abstract_content.empty?
               abstract << {
                 content: abstract_content,
@@ -121,7 +123,6 @@ module RelatonIso
         end
         [titles, abstract, langs]
       end
-      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
       # Returns available languages.
       # @param doc [Nokogiri::HTML::Document]
@@ -131,7 +132,7 @@ module RelatonIso
         lgs = [{ lang: "en" }]
         doc.css("li#lang-switcher ul li a").each do |lang_link|
           lang_path = lang_link.attr("href")
-          l = lang_path.match(%r{^\/(fr)\/})
+          l = lang_path.match(%r{^/(fr)/})
           lgs << { lang: l[1], path: lang_path } if l && (!lang || l[1] == lang)
         end
         lgs
@@ -168,67 +169,80 @@ module RelatonIso
       # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
       # Fetch docid.
-      # @param hit [Hash]
+      # @param doc [Nokogiri:HTML::Document]
+      # @param edition [String]
       # @param langs [Array<Hash>]
       # @return [Array<RelatonBib::DocumentIdentifier>]
-      def fetch_docid(hit, langs)
+      def fetch_docid(doc, edition, langs)
+        pubid = item_ref doc
         [
-          RelatonBib::DocumentIdentifier.new(id: hit["docRef"], type: "ISO"),
-          RelatonBib::DocumentIdentifier.new(id: fetch_urn(hit, langs),
-                                             type: "URN"),
+          RelatonBib::DocumentIdentifier.new(id: pubid, type: "ISO"),
+          RelatonBib::DocumentIdentifier.new(
+            id: fetch_urn(doc, pubid, edition, langs), type: "URN",
+          ),
         ]
       end
 
-      # @param hit [Hash]
+      # @param doc [Nokogiri:HTML::Document]
+      # @param pubid [String]
+      # @param edition [String]
       # @param langs [Array<Hash>]
       # @returnt [String]
-      def fetch_urn(hit, langs) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
-        orig = hit["docRef"].split(" ").first.downcase.split("/").join "-"
-        %r{(?<=)(?<type>DATA|GUIDE|ISP|IWA|PAS|R|TR|TS|TTA)} =~ hit["docRef"]
+      def fetch_urn(doc, pubid, edition, langs) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
+        orig = pubid.split.first.downcase.split("/").join "-"
+        %r{(?<=)(?<type>DATA|GUIDE|ISP|IWA|PAS|R|TR|TS|TTA)} =~ pubid
+        _, part, _year, corr, = IsoBibliography.ref_components pubid
         urn = "urn:iso:std:#{orig}"
         urn += ":#{type.downcase}" if type
-        urn += ":#{hit['docNumber']}"
-        urn += ":-#{hit['docPart']}" if hit["docPart"] && !hit["docPart"].empty?
-        urn += ":stage-#{hit['stageId']}"
-        urn += ":ed-#{hit['docEdition']}" if hit["docEdition"]
-        if hit["docElem"] && !hit["docElem"].empty? && hit["docElem"] != "0"
-          urn += ":#{hit['docElem'].downcase}:#{hit['docElemSeq']}"
+        urn += ":#{fetch_docnumber(doc)}"
+        urn += ":-#{part}" if part
+        urn += ":stage-#{stage_code(doc)}"
+        urn += ":ed-#{edition}" if edition
+        if corr
+          corrparts = corr.split
+          urn += ":#{corrparts[0].downcase}:#{corrparts[-1]}"
         end
-        urn += ":" + langs.map { |l| l[:lang] }.join(",")
+        urn += ":#{langs.map { |l| l[:lang] }.join(',')}"
         urn
       end
 
       def fetch_docnumber(doc)
-        id = doc.at("//nav[contains(@class, 'heading-condensed')]/h1")&.text
-        id&.match(/\d+/)&.to_s
+        item_ref(doc)&.match(/\d+/)&.to_s
       end
 
       # @param doc [Nokogiri::HTML::Document]
       def fetch_structuredidentifier(doc) # rubocop:disable Metrics/MethodLength
-        item_ref = doc.at("//nav[contains(@class, 'heading-condensed')]/h1")
-        unless item_ref
+        ref = item_ref doc
+        unless ref
           return RelatonIsoBib::StructuredIdentifier.new(
-            project_number: "?", part_number: "", prefix: nil, id: "?"
+            project_number: "?", part_number: "", prefix: nil, id: "?",
           )
         end
 
-        m = item_ref.text.match(/^(.*?\d+)-?((?<=-)\d+|)/)
+        m = ref.match(/^(.*?\d+)-?((?<=-)\d+|)/)
         RelatonIsoBib::StructuredIdentifier.new(
-          project_number: m[1], part_number: m[2], prefix: nil,
-          id: item_ref.text, type: "ISO"
+          project_number: m[1], part: m[2], type: "ISO",
         )
       end
 
+      def item_ref(doc)
+        doc.at("//nav[contains(@class, 'heading-condensed')]/h1")&.text
+      end
+
       # Fetch status.
       # @param doc [Nokogiri::HTML::Document]
       # @param status [String]
       # @return [Hash]
       def fetch_status(doc)
-        stg, substg = doc.at("//ul[@class='dropdown-menu']/li[@class='active']/a/span[@class='stage-code']")
-          .text.split "."
+        stg, substg = stage_code(doc).split "."
         RelatonBib::DocumentStatus.new(stage: stg, substage: substg)
       end
 
+      def stage_code(doc)
+        doc.at("//ul[@class='dropdown-menu']/li[@class='active']"\
+               "/a/span[@class='stage-code']").text
+      end
+
       # def stage(stg, substg)
       #   abbr = STGABBR[stg].is_a?(Hash) ? STGABBR[stg][substg] : STGABBR[stg]
       #   RelatonBib::DocumentStatus::Stage.new value: stg, abbreviation: abbr
@@ -241,13 +255,15 @@ module RelatonIso
         wg_link = doc.css("div.entry-name.entry-block a")[0]
         # wg_url = DOMAIN + wg_link['href']
         workgroup = wg_link.text.split "/"
+        type = workgroup[1]&.match(/^[A-Z]+/)&.to_s || "TC"
         {
           name: "International Organization for Standardization",
           abbreviation: "ISO",
           url: "www.iso.org",
           technical_committee: [{
-            name: wg_link.text + doc.css("div.entry-title")[0].text,
-            type: "TC",
+            name: doc.css("div.entry-title")[0].text,
+            identifier: wg_link.text,
+            type: type,
             number: workgroup[1]&.match(/\d+/)&.to_s&.to_i,
           }],
         }
@@ -274,10 +290,10 @@ module RelatonIso
           else
             a + r.css("a").map do |id|
               fref = RelatonBib::FormattedRef.new(
-                content: id.text, format: "text/plain"
+                content: id.text, format: "text/plain",
               )
               bibitem = RelatonIsoBib::IsoBibliographicItem.new(
-                formattedref: fref, date: date
+                formattedref: fref, date: date,
               )
               { type: type, bibitem: bibitem }
             end
@@ -311,7 +327,7 @@ module RelatonIso
       def fetch_title(doc, lang)
         content = doc.at(
           "//nav[contains(@class,'heading-condensed')]/h2 | "\
-          "//nav[contains(@class,'heading-condensed')]/h3"
+          "//nav[contains(@class,'heading-condensed')]/h3",
         )&.text&.gsub(/\u2014/, "-")
         return RelatonBib::TypedTitleStringCollection.new unless content
 
@@ -373,7 +389,7 @@ module RelatonIso
       def fetch_ics(doc)
         doc.xpath("//strong[contains(text(), "\
                   "'ICS')]/../following-sibling::dd/div/a").map do |i|
-          code = i.text.match(/[\d\.]+/).to_s.split "."
+          code = i.text.match(/[\d.]+/).to_s.split "."
           { field: code[0], group: code[1], subgroup: code[2] }
         end
       end
@@ -395,10 +411,10 @@ module RelatonIso
       end
 
       # Fetch copyright.
-      # @param ref [String]
       # @param doc [Nokogiri::HTML::Document]
       # @return [Array<Hash>]
-      def fetch_copyright(ref, doc)
+      def fetch_copyright(doc)
+        ref = item_ref doc
         owner_name = ref.match(/.*?(?=\s)/).to_s
         from = ref.match(/(?<=:)\d{4}/).to_s
         if from.empty?
@@ -408,5 +424,4 @@ module RelatonIso
       end
     end
   end
-  # rubocop:enable Metrics/ModuleLength
 end

data/lib/relaton_iso/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RelatonIso
-  VERSION = "1.8.0"
+  VERSION = "1.9.0"
 end

data/relaton_iso.gemspec

@@ -32,11 +32,15 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "rubocop-performance"
+  spec.add_development_dependency "rubocop-rails"
   # spec.add_development_dependency "ruby-debug-ide"
   spec.add_development_dependency "simplecov"
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
 
   # spec.add_dependency "relaton-iec", "~> 1.8.0"
-  spec.add_dependency "relaton-iso-bib", "~> 1.8.0"
+  spec.add_dependency "algolia"
+  spec.add_dependency "relaton-iso-bib", "~> 1.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-iso
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.9.0
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-05-17 00:00:00.000000000 Z
+date: 2021-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug
@@ -80,6 +80,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -122,20 +164,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: algolia
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: relaton-iso-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.9.0
 description: 'RelatonIso: retrieve ISO Standards for bibliographic use using the IsoBibliographicItem
   model'
 email:
@@ -165,9 +221,13 @@ files:
 - bin/nokogiri
 - bin/pry
 - bin/racc
+- bin/rackup
 - bin/rake
 - bin/rdebug-ide
 - bin/rspec
+- bin/rubocop
+- bin/ruby-parse
+- bin/ruby-rewrite
 - bin/safe_yaml
 - bin/setup
 - lib/relaton_iso.rb