relaton-itu 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a69976e4fa5be5f658de58234c3b2470702342f5
+  data.tar.gz: b8784a5c2fe9a57e3eea2ed104de573365b5be0f
+SHA512:
+  metadata.gz: 2fa691cc886fbcae97369fa4d61e016155194b820642bdc26554caa7639b1fa04a6c641f2b7a6ea7048afb36761cf6b12d32304a7a609a727f8095e1d1d4ac22
+  data.tar.gz: 2dae694690fd59635e27b9e7da5bd507dcbc49bdf06842cbdc24d893a2b1182e4432d92321ee368a4c3a111e78ef27a0a524ef1a39dbed2f9e98a3ea18a298dc

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.vscode/
+.rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,10 @@
+# This project follows the Ribose OSS style guide.
+# https://github.com/riboseinc/oss-guides
+# All project-specific additions and overrides should be specified in this file.
+
+inherit_from:
+  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
+AllCops:
+  TargetRubyVersion: 2.3
+Rails:
+  Enabled: true

data/.travis.yml

@@ -0,0 +1,15 @@
+language: ruby
+cache: bundler
+os:
+- linux
+- osx
+rvm:
+- 2.5
+- 2.4
+- ruby-head
+before_install:
+- gem install bundler -v 2.0.1
+- bundle update
+matrix:
+  allow_failures:
+    - rvm: ruby-head

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+gem 'vcr', git: 'https://github.com/vcr/vcr.git', ref: 'a0bb140184e8518baad640a705e04155911bf31f'
+
+# Specify your gem's dependencies in relaton_itu.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,93 @@
+GIT
+  remote: https://github.com/vcr/vcr.git
+  revision: a0bb140184e8518baad640a705e04155911bf31f
+  ref: a0bb140184e8518baad640a705e04155911bf31f
+  specs:
+    vcr (4.0.0)
+
+PATH
+  remote: .
+  specs:
+    relaton-itu (0.1.0)
+      iso-bib-item (~> 0.4.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.6.0)
+      public_suffix (>= 2.0.2, < 4.0)
+    byebug (11.0.1)
+    coderay (1.1.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    debase (0.2.2)
+      debase-ruby_core_source (>= 0.10.2)
+    debase-ruby_core_source (0.10.4)
+    diff-lcs (1.3)
+    docile (1.3.1)
+    equivalent-xml (0.6.0)
+      nokogiri (>= 1.4.3)
+    hashdiff (0.3.8)
+    iso-bib-item (0.4.5)
+      isoics (~> 0.1.6)
+      nokogiri (~> 1.8.4)
+      ruby_deep_clone (~> 0.8.0)
+    isoics (0.1.7)
+    json (2.2.0)
+    method_source (0.9.2)
+    mini_portile2 (2.3.0)
+    nokogiri (1.8.5)
+      mini_portile2 (~> 2.3.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    pry-byebug (3.7.0)
+      byebug (~> 11.0)
+      pry (~> 0.10)
+    public_suffix (3.0.3)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    ruby-debug-ide (0.6.1)
+      rake (>= 0.8.1)
+    ruby_deep_clone (0.8.0)
+    safe_yaml (1.0.5)
+    simplecov (0.16.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    webmock (3.5.1)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  debase
+  equivalent-xml (~> 0.6)
+  pry-byebug
+  rake (~> 10.0)
+  relaton-itu!
+  rspec (~> 3.0)
+  ruby-debug-ide
+  simplecov
+  vcr!
+  webmock
+
+BUNDLED WITH
+   2.0.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Andrei Kislichenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.adoc

@@ -0,0 +1,78 @@
+= RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem model
+
+image:https://img.shields.io/gem/v/relaton_bib.svg["Gem Version", link="https://rubygems.org/gems/relaton_bib"]
+image:https://travis-ci.com/metanorma/relaton_bib.svg?branch=master["Build Status", link="https://travis-ci.com/metanorma/relaton_bib"]
+image:https://ci.appveyor.com/api/projects/status/fd39m2762jo8ve04?svg=true["Appveyor Build Status", link="https://ci.appveyor.com/project/ribose/relaton_bib"]
+
+RelatonItu is a Ruby gem that implements the https://github.com/metanorma/metanorma-model-iso#iso-bibliographic-item[IsoBibliographicItem model].
+
+== Installation
+
+Add this line to your application's Gemfile:
+
+[source,ruby]
+----
+gem 'relaton_bib'
+----
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install relaton_bib
+
+== Usage
+
+=== Search for a standard using keywords
+
+[source,ruby]
+----
+require 'relaton_bib'
+
+hit_collection = RelatonItu::ItuBibliography.search("ITU-T L.163")
+=> [<RelatonItu::Hit:0x007f92b5d693c8 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T L.163 (11/2018)">,
+ <RelatonItu::Hit:0x007f92b5d69350 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T Z.161 (10/2018)">]
+ ...
+
+ item = hit_collection[1].fetch
+ => #<IsoBibItem::IsoBibliographicItem:0x007f92b5c723c0
+ ...
+----
+
+=== XML serialization
+[source,ruby]
+----
+item.to_xml
+=> "<bibitem type=\"international-standard\" id=\"ITU-TZ.161(10/2018)\">
+      <fetched>2019-03-01</fetched>
+      <title format=\"text/plain\" language=\"en\" script=\"Latn\">
+        ITU-T Z.161 (10/2018): Testing and Test Control Notation version 3: TTCN-3 core language
+      </title>
+      ...
+    </bibitem>"
+----
+
+=== Get code, and year
+[source,ruby]
+----
+RelatonItu::ItuBibliography.get("ITU-T L.163", "2018", {})
+fetching ITU-T L.163...
+=> #<IsoBibItem::IsoBibliographicItem:0x007f92b5d8bc20
+...
+----
+
+== Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+== Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/metanorma/relaton_bib.
+
+== License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/appveyor.yml

@@ -0,0 +1,28 @@
+version: '{build}'
+
+environment:
+  matrix:
+    - RUBY_VERSION: 25
+    - RUBY_VERSION: 24
+    - RUBY_VERSION: _trunk
+
+matrix:
+  allow_failures:
+    - RUBY_VERSION: _trunk
+
+install:
+  - ps: . { iwr -useb https://raw.githubusercontent.com/metanorma/metanorma-build-scripts/master/appveyor.ps1 } | iex
+  - refreshenv
+
+build_script:
+  - set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH%
+  - bundle update
+  - bundle install
+
+before_test:
+  - ruby -v
+  - gem -v
+  - bundle -v
+
+test_script:
+  - bundle exec rake

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "relaton_itu"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/itubib.gemspec

@@ -0,0 +1,40 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "relaton_itu/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "relaton-itu"
+  spec.version       = RelatonItu::VERSION
+  spec.authors       = ["Ribose Inc."]
+  spec.email         = ["open.source@ribose.com"]
+
+  spec.summary       = "RelatonItu: retrieve ITU Standards for bibliographic use "\
+                       "using the BibliographicItem model"
+  spec.description   = "RelatonItu: retrieve ITU Standards for bibliographic use "\
+                       "using the BibliographicItem model"
+  spec.homepage      = "https://github.com/metanorma/relaton-itu"
+  spec.license       = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path("..", __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "equivalent-xml", "~> 0.6"
+  spec.add_development_dependency "pry-byebug"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "vcr"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "ruby-debug-ide"
+  spec.add_development_dependency "debase"
+
+  spec.add_dependency "iso-bib-item", "~> 0.4.2"
+end

data/lib/relaton/processor.rb

@@ -0,0 +1,23 @@
+require "relaton/processor"
+
+module Relaton
+  module RelatonItu
+    class Processor < Relaton::Processor
+
+      def initialize
+        @short = :relaton_ite
+        @prefix = "ITU"
+        @defaultprefix = %r{^(ITU)}
+        @idtype = "ITU"
+      end
+
+      def get(code, date, opts)
+        ::RelatonItu::ItuBliography.get(code, date, opts)
+      end
+
+      def from_xml(xml)
+        IsoBibItem::XMLParser.from_xml xml
+      end
+    end
+  end
+end

data/lib/relaton_itu.rb

@@ -0,0 +1,12 @@
+require "relaton_itu/version"
+require 'relaton_itu/itu_bibliography'
+
+if defined? Relaton
+  require_relative 'relaton/processor'
+  Relaton::Registry.instance.register(Relaton::RelatonItu::Processor)
+end
+
+module RelatonItu
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/lib/relaton_itu/hit.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module RelatonItu
+  # Hit.
+  class Hit
+    # @return [Isobib::HitCollection]
+    attr_reader :hit_collection
+
+    # @return [Array<Hash>]
+    attr_reader :hit
+
+    # @param hit [Hash]
+    # @param hit_collection [Isobib:HitCollection]
+    def initialize(hit, hit_collection = nil)
+      @hit            = hit
+      @hit_collection = hit_collection
+    end
+
+    # Parse page.
+    # @return [Isobib::IsoBibliographicItem]
+    def fetch
+      @fetch ||= Scrapper.parse_page @hit
+    end
+
+    # @return [String]
+    def to_s
+      inspect
+    end
+
+    # @return [String]
+    def inspect
+      "<#{self.class}:#{format('%#.14x', object_id << 1)} "\
+      "@text=\"#{@hit_collection&.text}\" "\
+      "@fetched=\"#{!@fetch.nil?}\" "\
+      "@fullIdentifier=\"#{@fetch&.shortref(nil)}\" "\
+      "@title=\"#{@hit[:code]}\">"
+    end
+
+    # @return [String]
+    def to_xml(opts = {})
+      #if builder
+        #fetch.to_xml builder, opts
+      #else
+        builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+          fetch.to_xml xml, opts
+        end
+        builder.doc.root.to_xml
+      #end
+    end
+  end
+end

data/lib/relaton_itu/hit_collection.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'relaton_itu/hit'
+require "addressable/uri"
+require 'net/http'
+
+module RelatonItu
+  # Page of hit collection.
+  class HitCollection < Array
+
+    DOMAIN = 'https://www.itu.int'.freeze
+
+    # @return [TrueClass, FalseClass]
+    attr_reader :fetched
+
+    # @return [String]
+    attr_reader :text
+
+    # @return [String]
+    attr_reader :year
+
+    # @param ref_nbr [String]
+    # @param year [String]
+    def initialize(ref_nbr, year = nil) #(text, hit_pages = nil)
+      @text = ref_nbr
+      @year = year
+      from, to = nil
+      if year
+        from = Date.strptime year, '%Y'
+        to   = from.next_year.prev_day
+      end
+      url  = "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
+      params = {
+        "Input"=>"163",
+        "Start"=>0,
+        "Rows"=>10,
+        "SortBy"=>"RELEVANCE",
+        "ExactPhrase"=>false,
+        "CollectionName"=>"General",
+        "CollectionGroup"=>"Recommendations",
+        "Sector"=>"t",
+        "Criterias"=> [{
+          "Name"=>"Search in",
+          "Criterias" => [
+            {"Selected"=>false, "Value"=>"", "Label"=>"Name", "Target"=>"/name_s", "TypeName"=>"CHECKBOX", "GetCriteriaType"=>0},
+            {"Selected"=>false, "Value"=>"", "Label"=>"Short description", "Target"=>"/short_description_s", "TypeName"=>"CHECKBOX", "GetCriteriaType"=>0},
+            {"Selected"=>false, "Value"=>"", "Label"=>"File content", "Target"=>"/file", "TypeName"=>"CHECKBOX", "GetCriteriaType"=>0}
+          ],
+          "ShowCheckbox"=>true,
+          "Selected"=>false
+        }],
+        "Topics"=>"",
+        "ClientData"=>{"ip"=>""},
+        "Language"=>"en",
+        "IP"=>"",
+        "SearchType"=>"All"
+      }
+      data = { json: params.to_json }
+      resp  = Net::HTTP.post(URI(url), data.to_json, 'Content-Type' => 'application/json')
+      doc = JSON.parse resp.body
+      hits = doc['results'].map do |h|
+        code  = h['Media']['Name']
+        title = h['Title']
+        url   = h['Redirection']
+        Hit.new({ code: code, title: title, url: url }, self)
+      end
+      concat hits
+      @fetched = false
+    end
+
+    # @return [RelatonItu::HitCollection]
+    def fetch
+      workers = WorkersPool.new 4
+      workers.worker(&:fetch)
+      each do |hit|
+        workers << hit
+      end
+      workers.end
+      workers.result
+      @fetched = true
+      self
+    end
+
+    def to_s
+      inspect
+    end
+
+    def inspect
+      "<#{self.class}:#{format('%#.14x', object_id << 1)} @fetched=#{@fetched}>"
+    end
+  end
+end

data/lib/relaton_itu/itu_bibliography.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# require 'isobib/iso_bibliographic_item'
+require 'relaton_itu/scrapper'
+require 'relaton_itu/hit_collection'
+require "date"
+
+module RelatonItu
+  # Class methods for search ISO standards.
+  class ItuBibliography
+    class << self
+      # @param text [String]
+      # @return [RelatonItu::HitCollection]
+      def search(text, year = nil)
+        begin
+          HitCollection.new text, year
+        rescue
+          warn "Could not access http://www.itu.int"
+          []
+        end
+      end
+
+      # @param code [String] the ISO standard Code to look up (e..g "ISO 9000")
+      # @param year [String] the year the standard was published (optional)
+      # @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
+      # @return [String] Relaton XML serialisation of reference
+      def get(code, year = nil, opts = {})
+        if year.nil?
+          /^(?<code1>[^\s]+\s[^\s]+)\s\(\d{2}\/(?<year1>\d+)\)$/ =~ code
+          unless code1.nil?
+            code = code1
+            year = year1
+          end
+        end
+
+        code += '-1' if opts[:all_parts]
+        ret = itubib_get1(code, year, opts)
+        return nil if ret.nil?
+        ret.to_most_recent_reference unless year || opts[:keep_year]
+        ret.to_all_parts if opts[:all_parts]
+        ret
+      end
+
+      private
+
+      def fetch_ref_err(code, year, missed_years)
+        id = year ? "#{code}:#{year}" : code
+        warn "WARNING: no match found online for #{id}. "\
+          "The code must be exactly like it is on the standards website."
+        warn "(There was no match for #{year}, though there were matches "\
+          "found for #{missed_years.join(', ')}.)" unless missed_years.empty?
+        if /\d-\d/ =~ code
+          warn "The provided document part may not exist, or the document "\
+            "may no longer be published in parts."
+        else
+          warn "If you wanted to cite all document parts for the reference, "\
+            "use \"#{code} (all parts)\".\nIf the document is not a standard, "\
+            "use its document type abbreviation (TS, TR, PAS, Guide)."
+        end
+        nil
+      end
+
+      def fetch_pages(s, n)
+        workers = WorkersPool.new n
+        workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
+        s.each_with_index { |hit, i| workers << { i: i, hit: hit } }
+        workers.end
+        workers.result.sort { |x, y| x[:i] <=> y[:i] }.map { |x| x[:hit] }
+      end
+
+      def search_filter(code)
+        docidrx = %r{^ITU-T\s[^\s]+}
+        warn "fetching #{code}..."
+        result = search(code)
+        result.select do |i|
+          i.hit[:code] &&
+            i.hit[:code].match(docidrx).to_s == code
+        end
+      end
+
+      # Sort through the results from Isobib, fetching them three at a time,
+      # and return the first result that matches the code,
+      # matches the year (if provided), and which # has a title (amendments do not).
+      # Only expects the first page of results to be populated.
+      # Does not match corrigenda etc (e.g. ISO 3166-1:2006/Cor 1:2007)
+      # If no match, returns any years which caused mismatch, for error reporting
+      def isobib_results_filter(result, year)
+        missed_years = []
+        result.each_slice(3) do |s| # ISO website only allows 3 connections
+          fetch_pages(s, 3).each_with_index do |r, i|
+            return { ret: r } if !year
+            r.dates.select { |d| d.type == "published" }.each do |d|
+              return { ret: r } if year.to_i == d.on.year
+              missed_years << d.on.year
+            end
+          end
+        end
+        { years: missed_years }
+      end
+
+      def itubib_get1(code, year, opts)
+        result = search_filter(code) or return nil
+        ret = isobib_results_filter(result, year)
+        return ret[:ret] if ret[:ret]
+        fetch_ref_err(code, year, ret[:years])
+      end
+    end
+  end
+end

data/lib/relaton_itu/scrapper.rb

@@ -0,0 +1,292 @@
+# frozen_string_literal: true
+
+require 'iso_bib_item'
+require 'relaton_itu/hit'
+require 'nokogiri'
+require 'net/http'
+require 'relaton_itu/workers_pool'
+
+# Capybara.request_driver :poltergeist do |app|
+#   Capybara::Poltergeist::Driver.new app, js_errors: false
+# end
+# Capybara.default_driver = :poltergeist
+
+module RelatonItu
+  # Scrapper.
+  # rubocop:disable Metrics/ModuleLength
+  module Scrapper
+    DOMAIN = 'https://www.itu.int'
+
+    TYPES = {
+      'ISO'   => 'international-standard',
+      'TS'    => 'technicalSpecification',
+      'TR'    => 'technicalReport',
+      'PAS'   => 'publiclyAvailableSpecification',
+      'AWI'   => 'appruvedWorkItem',
+      'CD'    => 'committeeDraft',
+      'FDIS'  => 'finalDraftInternationalStandard',
+      'NP'    => 'newProposal',
+      'DIS'   => 'draftInternationalStandard',
+      'WD'    => 'workingDraft',
+      'R'     => 'recommendation',
+      'Guide' => 'guide'
+    }.freeze
+
+    class << self
+      # @param text [String]
+      # @return [Array<Hash>]
+      # def get(text)
+      #   iso_workers = WorkersPool.new 4
+      #   iso_workers.worker { |hit| iso_worker(hit, iso_workers) }
+      #   algolia_workers = start_algolia_search(text, iso_workers)
+      #   iso_docs = iso_workers.result
+      #   algolia_workers.end
+      #   algolia_workers.result
+      #   iso_docs
+      # end
+
+      # Parse page.
+      # @param hit [Hash]
+      # @return [Hash]
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def parse_page(hit_data)
+        doc = get_page hit_data[:url]
+
+        # Fetch edition.
+        edition = doc.at("//table/tr/td/span[contains(@id, 'Label8')]/b").text
+
+        IsoBibItem::IsoBibliographicItem.new(
+          docid:        fetch_docid(hit_data[:code]),
+          edition:      edition,
+          language:     ['en'],
+          script:       ['Latn'],
+          titles:       fetch_titles(hit_data),
+          type:         fetch_type(doc),
+          docstatus:    fetch_status(doc),
+          ics:          [], # fetch_ics(doc),
+          dates:        fetch_dates(doc),
+          contributors: fetch_contributors(hit_data[:code]),
+          workgroup:    fetch_workgroup(doc),
+          abstract:     fetch_abstract(doc),
+          copyright:    fetch_copyright(hit_data[:code], doc),
+          link:         fetch_link(doc, hit_data[:url]),
+          relations:    fetch_relations(doc)
+        )
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+      private
+
+      # Fetch abstracts.
+      # @param doc [Nokigiri::HTML::Document]
+      # @return [Array<Array>]
+      def fetch_abstract(doc)
+        abstract_url = doc.at('//table/tr/td/span[contains(@id, "lbl_dms")]/div')
+        return [] unless abstract_url
+
+        url = abstract_url[:onclick].match(/https?[^']+/).to_s
+        d = Nokogiri::HTML Net::HTTP.get(URI(url))
+        abstract_content = d.css('p.MsoNormal').text.gsub(/\r\n/, '')
+          .gsub(/\s{2,}/, ' ').gsub(/\u00a0/, '')
+
+        [{
+          content:  abstract_content,
+          language: 'en',
+          script:   'Latn'
+        }]
+      end
+
+      # Get langs.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      # def langs(doc)
+      #   lgs = [{ lang: 'en' }]
+      #   doc.css('ul#lang-switcher ul li a').each do |lang_link|
+      #     lang_path = lang_link.attr('href')
+      #     lang = lang_path.match(%r{^\/(fr)\/})
+      #     lgs << { lang: lang[1], path: lang_path } if lang
+      #   end
+      #   lgs
+      # end
+
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      # Get page.
+      # @param path [String] page's path
+      # @return [Array<Nokogiri::HTML::Document, String>]
+      def get_page(url)
+        uri = URI url
+        resp = Net::HTTP.get_response(uri)#.encode("UTF-8")
+        while resp.code == '301' || resp.code == '302' || resp.code == '303'
+          uri = URI resp['location']
+          resp = Net::HTTP.get_response(uri)#.encode("UTF-8")
+        end
+        Nokogiri::HTML(resp.body)
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+      # Fetch docid.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Hash]
+      def fetch_docid(code)
+        m = code.match(/(?<=\s)(?<project>[^\s]+)-?(?<part>(?<=-)\d+|)-?(?<subpart>(?<=-)\d+|)/)
+        {
+          project_number: m[:project],
+          part_number: m[:part],
+          subpart_number: m[:subpart],
+          prefix: nil,
+          type: 'ITU',
+          id: code
+        }
+      end
+
+      # Fetch status.
+      # @param doc [Nokogiri::HTML::Document]
+      # @param status [String]
+      # @return [Hash]
+      def fetch_status(doc)
+        s = doc.at("//table/tr/td/span[contains(@id, 'Label7')]").text
+        if s == 'In force'
+          status   = 'Published'
+          stage    = '60'
+          substage = '60'
+        else
+          status   = 'Withdrawal'
+          stage    = '95'
+          substage = '99'
+        end
+        { status: status, stage: stage, substage: substage }
+      end
+
+      # Fetch workgroup.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Hash]
+      def fetch_workgroup(doc)
+        wg = doc.at('//table/tr/td/span[contains(@id, "Label8")]/a').text
+        { name:                'International Telecommunication Union',
+          abbreviation:        'ITU',
+          url:                 'www.itu.int',
+          technical_committee: {
+            name:   wg,
+            type:   'technicalCommittee',
+            number: wg.match(/\d+/)&.to_s&.to_i
+          } }
+      end
+
+      # Fetch relations.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      # rubocop:disable Metrics/MethodLength
+      def fetch_relations(doc)
+        doc.xpath('//div[contains(@id, "tab_sup")]//table/tr[position()>2]').map do |r|
+          r_type = r.at('./td/span[contains(@id, "Label4")]/nobr').text.downcase
+          type = case r_type
+                 when 'in force' then 'published'
+                 else r_type
+                 end
+          ref = r.at('./td/span[contains(@id, "title_e")]/nobr/a')
+          url = DOMAIN + ref[:href].sub(/^\./, '/ITU-T/recommendations')
+          { type: type, identifier: ref.text, url: url }
+        end
+      end
+
+      # Fetch type.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [String]
+      def fetch_type(doc)
+        'international-standard'
+      end
+
+      # Fetch titles.
+      # @param hit_data [Hash]
+      # @return [Array<Hash>]
+      def fetch_titles(hit_data)
+        titles = hit_data[:title].split ' - '
+        case titles.size
+        when 0
+          intro, main, part = nil, "", nil
+        when 1
+          intro, main, part = nil, titles[0], nil
+        when 2
+          if /^(Part|Partie) \d+:/ =~ titles[1]
+            intro, main, part = nil, titles[0], titles[1]
+          else
+            intro, main, part = titles[0], titles[1], nil
+          end
+        when 3
+          intro, main, part = titles[0], titles[1], titles[2]
+        else
+          intro, main, part = titles[0], titles[1], titles[2..-1]&.join(" -- ")
+        end
+        [{
+          title_intro: intro,
+          title_main:  main,
+          title_part:  part,
+          language:    'en',
+          script:      'Latn'
+        }]
+      end
+
+      # Fetch dates
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      def fetch_dates(doc)
+        dates = []
+        publish_date = doc.at("//table/tr/td/span[contains(@id, 'Label5')]").text
+        unless publish_date.empty?
+          dates << { type: 'published', on: publish_date }
+        end
+        dates
+      end
+
+      # Fetch contributors
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      def fetch_contributors(code)
+        abbrev = code.sub(/-\w\s.*/, '')
+        case abbrev
+        when 'ITU'
+          name = 'International Telecommunication Union'
+          url = 'www.itu.int'
+        end
+        [{ entity: { name: name, url: url, abbreviation: abbrev }, roles: ['publisher'] }]
+      end
+
+      # Fetch ICS.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      # def fetch_ics(doc)
+      #   doc.xpath('//th[contains(text(), "ICS")]/following-sibling::td/a').map do |i|
+      #     code = i.text.match(/[\d\.]+/).to_s.split '.'
+      #     { field: code[0], group: code[1], subgroup: code[2] }
+      #   end
+      # end
+
+      # Fetch links.
+      # @param doc [Nokogiri::HTML::Document]
+      # @param url [String]
+      # @return [Array<Hash>]
+      def fetch_link(doc, url)
+        links = [{ type: 'src', content: url }]
+        obp_elms = doc.at('//table/tr/td/span[contains(@id, "Label4")]/a')
+        links << { type: 'obp', content: DOMAIN + obp_elms[:href] } if obp_elms
+        links
+      end
+
+      # Fetch copyright.
+      # @param code [String]
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Hash]
+      def fetch_copyright(code, doc)
+        abbreviation = code.match(/^[^-]+/).to_s
+        case abbreviation
+        when 'ITU'
+          name = 'International Telecommunication Union'
+          url = 'www.itu.int'
+        end
+        from = doc.at("//table/tr/td/span[contains(@id, 'Label5')]").text
+        { owner: { name: name, abbreviation: abbreviation, url: url }, from: from }
+      end
+    end
+  end
+  # rubocop:enable Metrics/ModuleLength
+end

data/lib/relaton_itu/version.rb

@@ -0,0 +1,3 @@
+module RelatonItu
+  VERSION = "0.1.0".freeze
+end

data/lib/relaton_itu/workers_pool.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Workers poll.
+class WorkersPool
+  attr_accessor :nb_hits
+
+  def initialize(num_workers = 2)
+    @num_workers = num_workers < 2 ? 2 : num_workers
+    @queue = SizedQueue.new(num_workers * 2)
+    @result = []
+    @nb_hits = 0
+  end
+
+  def worker(&block)
+    @threads = Array.new @num_workers do
+      Thread.new do
+        until (item = @queue.pop) == :END
+          @result << yield(item) if block
+        end
+      end
+    end
+  end
+
+  def result
+    @threads.each(&:join)
+    @result
+  end
+
+  def <<(item)
+    @queue << item
+    self
+  end
+
+  def end
+    @num_workers.times { @queue << :END }
+  end
+
+  def size
+    @result.size
+  end
+end

metadata

@@ -0,0 +1,221 @@
+--- !ruby/object:Gem::Specification
+name: relaton-itu
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ribose Inc.
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-05-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: equivalent-xml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-debug-ide
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debase
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: iso-bib-item
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+description: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem
+  model'
+email:
+- open.source@ribose.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.adoc
+- Rakefile
+- appveyor.yml
+- bin/console
+- bin/setup
+- itubib.gemspec
+- lib/relaton/processor.rb
+- lib/relaton_itu.rb
+- lib/relaton_itu/hit.rb
+- lib/relaton_itu/hit_collection.rb
+- lib/relaton_itu/itu_bibliography.rb
+- lib/relaton_itu/scrapper.rb
+- lib/relaton_itu/version.rb
+- lib/relaton_itu/workers_pool.rb
+homepage: https://github.com/metanorma/relaton-itu
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem
+  model'
+test_files: []