iecbib 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 26a37510b59074822007109b47913b77db41dc9caea1c15f6f4392f5ff092d77
+  data.tar.gz: f220aca85402102dfebf9c1a2b3904a20c3ba45e6e436d02e25c424281e15788
+SHA512:
+  metadata.gz: 18a68279b7bc1cdf52b55c365fb3bb57f048af3ecc4525205ac160627a8172e2b085ae980940091787582d0c0504eddc1d377c213128bd4281fb0677d47a23bd
+  data.tar.gz: 7047b8f2a1d1344528c6ba51c978becc44db6418354960daf7d9155c6076f1c5da9c4e4605cfdc9e3989be8ee4486e2f74cbf742c8b45d075093c8c4df839a3b

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,4 @@
+AllCops:
+  TargetRubyVersion: 2.4.0
+Metrics/BlockLength:
+  ExcludedMethods: ['describe', 'context']

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.0
+before_install: gem install bundler -v 1.16.1

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in iecbib.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,80 @@
+PATH
+  remote: .
+  specs:
+    iecbib (0.1.1)
+      iso-bib-item (~> 0.3.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    byebug (10.0.2)
+    coderay (1.1.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    docile (1.3.1)
+    equivalent-xml (0.6.0)
+      nokogiri (>= 1.4.3)
+    hashdiff (0.3.7)
+    iso-bib-item (0.3.0)
+      isoics (~> 0.1.6)
+      nokogiri (~> 1.8.4)
+      ruby_deep_clone (~> 0.8.0)
+    isoics (0.1.7)
+    json (2.1.0)
+    method_source (0.9.0)
+    mini_portile2 (2.3.0)
+    nokogiri (1.8.4)
+      mini_portile2 (~> 2.3.0)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    pry-byebug (3.6.0)
+      byebug (~> 10.0)
+      pry (~> 0.10)
+    public_suffix (3.0.3)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    ruby_deep_clone (0.8.0)
+    safe_yaml (1.0.4)
+    simplecov (0.16.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    vcr (4.0.0)
+    webmock (3.4.2)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  equivalent-xml (~> 0.6)
+  iecbib!
+  pry-byebug
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  simplecov
+  vcr
+  webmock
+
+BUNDLED WITH
+   1.16.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Andrei Kislichenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.adoc

@@ -0,0 +1,40 @@
+= Iecbib
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/iecbib`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+== Installation
+
+Add this line to your application's Gemfile:
+
+[source,ruby]
+----
+gem 'iecbib'
+----
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install iecbib
+
+== Usage
+
+TODO: Write usage instructions here
+
+== Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+== Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/iecbib.
+
+== License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "iecbib"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/iecbib.gemspec

@@ -0,0 +1,36 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "iecbib/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "iecbib"
+  spec.version       = Iecbib::VERSION
+  spec.authors       = ["Ribose Inc."]
+  spec.email         = ["open.source@ribose.com"]
+
+  spec.summary       = "IsoBib: retrieve ISO Standards for bibliographic use "\
+                       "using the BibliographicItem model"
+  spec.description   = "IsoBib: retrieve ISO Standards for bibliographic use "\
+                       "using the BibliographicItem model"
+  spec.homepage      = "https://github.com/riboseinc/isobib"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'equivalent-xml', '~> 0.6'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'vcr'
+  spec.add_development_dependency 'webmock'
+
+  spec.add_dependency 'iso-bib-item', '~> 0.3.0'
+end

data/lib/iecbib.rb

@@ -0,0 +1,10 @@
+require "iecbib/version"
+require 'iecbib/iec_bibliography'
+if defined? Relaton
+  require_relative 'relaton/processor'
+  Relaton::Registry.instance.register(Relaton::Iecbib::Processor)
+end
+
+module Iecbib
+  # Your code goes here...
+end

data/lib/iecbib/hit.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Iecbib
+  # Hit.
+  class Hit
+    # @return [Isobib::HitCollection]
+    attr_reader :hit_collection
+
+    # @return [Array<Hash>]
+    attr_reader :hit
+
+    # @param hit [Hash]
+    # @param hit_collection [Isobib:HitCollection]
+    def initialize(hit, hit_collection = nil)
+      @hit            = hit
+      @hit_collection = hit_collection
+    end
+
+    # Parse page.
+    # @return [Isobib::IsoBibliographicItem]
+    def fetch
+      @fetch ||= Scrapper.parse_page @hit
+    end
+
+    # @return [String]
+    def to_s
+      inspect
+    end
+
+    # @return [String]
+    def inspect
+      "<#{self.class}:#{format('%#.14x', object_id << 1)} "\
+      "@text=\"#{@hit_collection&.text}\" "\
+      "@fetched=\"#{!@fetch.nil?}\" "\
+      "@fullIdentifier=\"#{@fetch&.shortref(nil)}\" "\
+      "@title=\"#{@hit[:code]}\">"
+    end
+
+    # @return [String]
+    def to_xml(opts = {})
+      #if builder
+        #fetch.to_xml builder, opts
+      #else
+        builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+          fetch.to_xml xml, opts
+        end
+        builder.doc.root.to_xml
+      #end
+    end
+  end
+end

data/lib/iecbib/hit_collection.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'iecbib/hit'
+
+module Iecbib
+  # Page of hit collection.
+  class HitCollection < Array
+
+    DOMAIN = 'https://webstore.iec.ch'
+
+    # @return [TrueClass, FalseClass]
+    attr_reader :fetched
+
+    # @return [String]
+    attr_reader :text
+
+    # @return [String]
+    attr_reader :year
+
+    # @param ref_nbr [String]
+    # @param year [String]
+    def initialize(ref_nbr, year = nil) #(text, hit_pages = nil)
+      @text = ref_nbr
+      @year = year
+      from, to = nil
+      if year
+        from = Date.strptime year, '%Y'
+        to   = from.next_year.prev_day
+      end
+      url  = "#{DOMAIN}/searchkey&RefNbr=#{ref_nbr}&From=#{from}&To=#{to}&start=1"
+      doc  = Nokogiri::HTML OpenURI.open_uri(Addressable::URI.parse(url).normalize)
+      hits = doc.css('ul.search-results > li').map do |h|
+        link  = h.at('a[@href!="#"]')
+        code  = link.text.tr [194, 160].pack('c*').force_encoding('UTF-8'), ''
+        title = h.xpath('text()').text.gsub(/[\r\n]/, '')
+        url   = DOMAIN + link[:href]
+        Hit.new({ code: code, title: title, url: url }, self)
+      end
+      concat hits
+      # concat(hits.map { |h| Hit.new(h, self) })
+      @fetched = false
+      # @hit_pages = hit_pages
+    end
+
+    # @return [Iecbib::HitCollection]
+    def fetch
+      workers = WorkersPool.new 4
+      workers.worker(&:fetch)
+      each do |hit|
+        workers << hit
+      end
+      workers.end
+      workers.result
+      @fetched = true
+      self
+    end
+
+    def to_s
+      inspect
+    end
+
+    def inspect
+      "<#{self.class}:#{format('%#.14x', object_id << 1)} @fetched=#{@fetched}>"
+    end
+  end
+end

data/lib/iecbib/iec_bibliography.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+# require 'isobib/iso_bibliographic_item'
+require 'iecbib/scrapper'
+require 'iecbib/hit_collection'
+
+module Iecbib
+  # Class methods for search ISO standards.
+  class IecBibliography
+    class << self
+      # @param text [String]
+      # @return [Iecbib::HitCollection]
+      def search(text, year = nil)
+        HitCollection.new text, year
+      end
+
+      # @param text [String]
+      # @return [Array<IsoBibliographicItem>]
+      def search_and_fetch(text, year = nil)
+        Scrapper.get(text, year)
+      end
+
+      # @param code [String] the ISO standard Code to look up (e..g "ISO 9000")
+      # @param year [String] the year the standard was published (optional)
+      # @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
+      # @return [String] Relaton XML serialisation of reference
+      def get(code, year, opts)
+        return iev if code.casecmp('IEV') == 0
+        code += '-1' if opts[:all_parts]
+        ret = iecbib_get1(code, year, opts)
+        return nil if ret.nil?
+        ret.to_most_recent_reference unless year
+        ret.to_all_parts if opts[:all_parts]
+        ret # .to_xml
+      end
+
+      private
+
+      def fetch_ref_err(code, year, missed_years)
+        id = year ? "#{code}:#{year}" : code
+        warn "WARNING: no match found online for #{id}. "\
+          "The code must be exactly like it is on the standards website."
+        warn "(There was no match for #{year}, though there were matches "\
+          "found for #{missed_years.join(', ')}.)" unless missed_years.empty?
+        if /\d-\d/ =~ code
+          warn "The provided document part may not exist, or the document "\
+            "may no longer be published in parts."
+        else
+          warn "If you wanted to cite all document parts for the reference, "\
+            "use \"#{code} (all parts)\".\nIf the document is not a standard, "\
+            "use its document type abbreviation (TS, TR, PAS, Guide)."
+        end
+        nil
+      end
+
+      def fetch_pages(s, n)
+        workers = WorkersPool.new n
+        workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
+        s.each_with_index { |hit, i| workers << { i: i, hit: hit } }
+        workers.end
+        workers.result.sort { |x, y| x[:i] <=> y[:i] }.map { |x| x[:hit] }
+      end
+
+      def isobib_search_filter(code)
+        docidrx = %r{^(ISO|IEC)[^0-9]*\s[0-9-]+}
+        corrigrx = %r{^(ISO|IEC)[^0-9]*\s[0-9-]+:[0-9]+/}
+        warn "fetching #{code}..."
+        result = search(code)
+        result.select do |i|
+          i.hit[:code] &&
+            i.hit[:code].match(docidrx).to_s == code &&
+            corrigrx !~ i.hit[:code]
+        end
+      end
+
+      def iev(code = "IEC 60050")
+        IsoBibItem.from_xml(<<~"END")
+          <bibitem type="international-standard" id="IEV">
+  <title format="text/plain" language="en" script="Latn">International Electrotechnical Vocabulary</title>
+  <link type="src">http://www.electropedia.org</link>
+  <docidentifier>#{code}:2011</docidentifier>
+  <date type="published"><on>2011</on></date>
+  <contributor>
+    <role type="publisher"/>
+    <organization>
+      <name>International Electrotechnical Commission</name>
+      <abbreviation>IEC</abbreviation>
+      <uri>www.iec.ch</uri>
+    </organization>
+  </contributor>
+  <language>en</language> <language>fr</language>
+  <script>Latn</script>
+  <status> <stage>60</stage> </status>
+  <copyright>
+    <from>2018</from>
+    <owner>
+      <organization>
+      <name>International Electrotechnical Commission</name>
+      <abbreviation>IEC</abbreviation>
+      <uri>www.iec.ch</uri>
+      </organization>
+    </owner>
+  </copyright>
+</bibitem>
+        END
+      end
+
+      # Sort through the results from Isobib, fetching them three at a time,
+      # and return the first result that matches the code,
+      # matches the year (if provided), and which # has a title (amendments do not).
+      # Only expects the first page of results to be populated.
+      # Does not match corrigenda etc (e.g. ISO 3166-1:2006/Cor 1:2007)
+      # If no match, returns any years which caused mismatch, for error reporting
+      def isobib_results_filter(result, year)
+        missed_years = []
+        result.each_slice(3) do |s| # ISO website only allows 3 connections
+          fetch_pages(s, 3).each_with_index do |r, i|
+            return { ret: r } if !year
+            r.dates.select { |d| d.type == "published" }.each do |d|
+              return { ret: r } if year.to_i == d.on.year
+              missed_years << d.on.year
+            end
+          end
+        end
+        { years: missed_years }
+      end
+
+      def iecbib_get1(code, year, opts)
+        return iev if code.casecmp("IEV") == 0
+        result = isobib_search_filter(code) or return nil
+        ret = isobib_results_filter(result, year)
+        return ret[:ret] if ret[:ret]
+        fetch_ref_err(code, year, ret[:years])
+      end
+    end
+  end
+end

data/lib/iecbib/scrapper.rb

@@ -0,0 +1,392 @@
+# frozen_string_literal: true
+
+require 'iso_bib_item'
+require 'iecbib/hit'
+require 'nokogiri'
+require 'net/http'
+require 'iecbib/workers_pool'
+
+# Capybara.request_driver :poltergeist do |app|
+#   Capybara::Poltergeist::Driver.new app, js_errors: false
+# end
+# Capybara.default_driver = :poltergeist
+
+module Iecbib
+  # Scrapper.
+  # rubocop:disable Metrics/ModuleLength
+  module Scrapper
+    DOMAIN = 'https://webstore.iec.ch'
+
+    TYPES = {
+      'ISO'   => 'international-standard',
+      'TS'    => 'technicalSpecification',
+      'TR'    => 'technicalReport',
+      'PAS'   => 'publiclyAvailableSpecification',
+      'AWI'   => 'appruvedWorkItem',
+      'CD'    => 'committeeDraft',
+      'FDIS'  => 'finalDraftInternationalStandard',
+      'NP'    => 'newProposal',
+      'DIS'   => 'draftInternationalStandard',
+      'WD'    => 'workingDraft',
+      'R'     => 'recommendation',
+      'Guide' => 'guide'
+    }.freeze
+
+    class << self
+      # @param text [String]
+      # @return [Array<Hash>]
+      # def get(text)
+      #   iso_workers = WorkersPool.new 4
+      #   iso_workers.worker { |hit| iso_worker(hit, iso_workers) }
+      #   algolia_workers = start_algolia_search(text, iso_workers)
+      #   iso_docs = iso_workers.result
+      #   algolia_workers.end
+      #   algolia_workers.result
+      #   iso_docs
+      # end
+
+      # Parse page.
+      # @param hit [Hash]
+      # @return [Hash]
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def parse_page(hit_data)
+        doc = get_page hit_data[:url]
+
+        # Fetch edition.
+        edition = doc.at("//th[contains(., 'Edition')]/following-sibling::td/span").text
+
+        status, relations = fetch_status_relations hit_data[:url]
+
+        IsoBibItem::IsoBibliographicItem.new(
+          docid:        fetch_docid(doc),
+          edition:      edition,
+          language:     ['en'],
+          script:       ['Latn'],
+          titles:       fetch_titles(hit_data),
+          type:         fetch_type(doc),
+          docstatus:    status,
+          ics:          fetch_ics(doc),
+          dates:        fetch_dates(doc),
+          contributors: fetch_contributors(hit_data[:code]),
+          workgroup:    fetch_workgroup(doc),
+          abstract:     fetch_abstract(doc),
+          copyright:    fetch_copyright(hit_data[:code], doc),
+          link:         fetch_link(doc, hit_data[:url]),
+          relations:    relations
+        )
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+      private
+
+      # Start search workers.
+      # @param text[String]
+      # @param iec_workers [Isobib::WorkersPool]
+      # @reaturn [Isobib::WorkersPool]
+      # def start_algolia_search(text, iec_workers)
+      #   index = Algolia::Index.new 'all_en'
+      #   workers = WorkersPool.new
+      #   workers.worker do |page|
+      #     algolia_worker(index, text, page, workers, iec_workers)
+      #   end
+
+      #   # Add first page so search worker will start.
+      #   workers << 0
+      # end
+
+      # Fetch ISO documents.
+      # @param hit [Hash]
+      # @param isiso_workers [Isobib::WorkersPool]
+      # def iso_worker(hit, iso_workers)
+      #   print "Parse #{iso_workers.size} of #{iso_workers.nb_hits}  \r"
+      #   parse_page hit
+      # end
+
+      # Fetch hits from algolia search service.
+      # @param index[Algolia::Index]
+      # @param text [String]
+      # @param page [Integer]
+      # @param algolia_workers [Isobib::WorkersPool]
+      # @param isiso_workers [Isobib::WorkersPool]
+      # def algolia_worker(index, text, page, algolia_workers, iso_workers)
+      #   res = index.search text, facetFilters: ['category:standard'], page: page
+      #   next_page = res['page'] + 1
+      #   algolia_workers << next_page if next_page < res['nbPages']
+      #   res['hits'].each do |hit|
+      #     iso_workers.nb_hits = res['nbHits']
+      #     iso_workers << hit
+      #   end
+      #   iso_workers.end unless next_page < res['nbPages']
+      # end
+
+      # Fetch abstracts.
+      # @param doc [Nokigiri::HTML::Document]
+      # @return [Array<Array>]
+      def fetch_abstract(doc)
+        abstract_content = doc.at('//div[@itemprop="description"]').text
+        [{
+          content:  abstract_content,
+          language: 'en',
+          script:   'Latn'
+        }]
+      end
+
+      # Get langs.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      # def langs(doc)
+      #   lgs = [{ lang: 'en' }]
+      #   doc.css('ul#lang-switcher ul li a').each do |lang_link|
+      #     lang_path = lang_link.attr('href')
+      #     lang = lang_path.match(%r{^\/(fr)\/})
+      #     lgs << { lang: lang[1], path: lang_path } if lang
+      #   end
+      #   lgs
+      # end
+
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      # Get page.
+      # @param path [String] page's path
+      # @return [Array<Nokogiri::HTML::Document, String>]
+      def get_page(url)
+        uri = URI url
+        resp = Net::HTTP.get_response(uri)#.encode("UTF-8")
+        if resp.code == '301'
+          path = resp['location']
+          url = DOMAIN + path
+          uri = URI url
+          resp = Net::HTTP.get_response(uri)#.encode("UTF-8")
+        end
+        # n = 0
+        # while resp.body !~ /<strong/ && n < 10
+        #   resp = Net::HTTP.get_response(uri)#.encode("UTF-8")
+        #   n += 1
+        # end
+        Nokogiri::HTML(resp.body)
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+      # Fetch docid.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Hash]
+      def fetch_docid(doc)
+        item_ref = doc.at("//span[@itemprop='productID']")
+        unless item_ref
+          return { project_number: '?', part_number: '', prefix: nil, id: '?' }
+        end
+        m = item_ref.text.match(/(?<=\s)(?<project>\d+)-?(?<part>(?<=-)\d+|)-?(?<subpart>(?<=-)\d+|)/)
+        {
+          project_number: m[:project],
+          part_number: m[:part],
+          subpart_number: m[:subpart],
+          prefix: nil,
+          type: 'IEC',
+          id: item_ref.text
+        }
+      end
+
+      # Fetch status.
+      # @param doc [Nokogiri::HTML::Document]
+      # @param status [String]
+      # @return [Hash]
+      def fetch_status(doc)
+        wip = doc.at('//ROW[STATUS[.="PREPARING"]]')
+        if wip
+          statuses = YAML.load_file 'lib/iecbib/statuses.yml'
+          s = wip.at('STAGE').text
+          stage, substage = statuses[s]['stage'].split '.'
+          status = statuses[s]['status']
+        else
+          status   = 'Published'
+          stage    = '60'
+          substage = '60'
+        end
+        { status: status, stage: stage, substage: substage }
+      end
+
+      # Fetch workgroup.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Hash]
+      def fetch_workgroup(doc)
+        wg = doc.at('//th/abbr[.="TC"]/../following-sibling::td/a').text
+        { name:                'International Electrotechnical Commission',
+          abbreviation:        'IEC',
+          url:                 'webstore.iec.ch',
+          technical_committee: {
+            name:   wg,
+            type:   'technicalCommittee',
+            number: wg.match(/\d+/)&.to_s&.to_i
+          } }
+      end
+
+      # Fetch relations.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      # rubocop:disable Metrics/MethodLength
+      def fetch_relations(doc)
+        doc.xpath('//ROW[STATUS[.!="PREPARING"]][STATUS[.!="PUBLISHED"]]').map do |r|
+          r_type = r.at('STATUS').text.downcase
+          type = case r_type
+                #  when 'published' then 'obsoletes' # Valid
+                 when 'revised', 'replaced' then 'updates'
+                 when 'withdrawn' then 'obsoletes'
+                 else r_type
+                 end
+          url = DOMAIN + '/publication/' + r.at('PUB_ID').text
+          { type: type, identifier: r.at('FULL_NAME').text, url: url }
+        end
+      end
+
+      def fetch_status_relations(url)
+        pubid = url.match(/\d+$/).to_s
+        uri = URI DOMAIN + '/webstore/webstore.nsf/AjaxRequestXML?'\
+        'Openagent&url=http://www.iec.ch/dyn/www/f?'\
+        'p=103:390:::::P390_PUBLICATION_ID:' + pubid
+        resp = Net::HTTP.get_response uri
+        doc = Nokogiri::XML resp.body
+        status = fetch_status doc
+        relations = fetch_relations doc
+        [status, relations]
+        # doc.css('ul.steps li').inject([]) do |a, r|
+        #   r_type = r.css('strong').text
+        #   type = case r_type
+        #          when 'Previously', 'Will be replaced by' then 'obsoletes'
+        #          when 'Corrigenda/Amendments', 'Revised by', 'Now confirmed'
+        #            'updates'
+        #          else r_type
+        #          end
+        #   if ['Now', 'Now under review'].include? type
+        #     a
+        #   else
+        #     a + r.css('a').map do |id|
+        #       { type: type, identifier: id.text, url: id['href'] }
+        #     end
+        #   end
+        # end
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      # Fetch type.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [String]
+      def fetch_type(doc)
+        doc.at('//th[contains(., "Publication type")]/following-sibling::td/span')
+           .text.downcase.tr ' ', '-'
+        # type_match = title.match(%r{^(ISO|IWA|IEC)(?:(/IEC|/IEEE|/PRF|
+        #   /NP)*\s|/)(TS|TR|PAS|AWI|CD|FDIS|NP|DIS|WD|R|Guide|(?=\d+))}x)
+        # #return "international-standard" if type_match.nil?
+        # if TYPES[type_match[2]]
+        #   TYPES[type_match[2]]
+        # elsif type_match[1]
+        # elsif type_match[1] == 'ISO'
+        #   'international-standard'
+        # elsif type_match[1] == 'IWA'
+        #   'international-workshop-agreement'
+        # end
+        # # rescue => _e
+        # #   puts 'Unknown document type: ' + title
+      end
+
+      # Fetch titles.
+      # @param hit_data [Hash]
+      # @return [Array<Hash>]
+      def fetch_titles(hit_data)
+        titles = hit_data[:title].split ' - '
+        case titles.size
+        when 0
+          intro, main, part = nil, "", nil
+        when 1
+          intro, main, part = nil, titles[0], nil
+        when 2
+          if /^(Part|Partie) \d+:/ =~ titles[1]
+            intro, main, part = nil, titles[0], titles[1]
+          else
+            intro, main, part = titles[0], titles[1], nil
+          end
+        when 3
+          intro, main, part = titles[0], titles[1], titles[2]
+        else
+          intro, main, part = titles[0], titles[1], titles[2..-1]&.join(" -- ")
+        end
+        [{
+          title_intro: intro,
+          title_main:  main,
+          title_part:  part,
+          language:    'en',
+          script:      'Latn'
+        }]
+      end
+
+      # Return ISO script code.
+      # @param lang [String]
+      # @return [String]
+      # def script(lang)
+      #   case lang
+      #   when 'en', 'fr' then 'Latn'
+      #   end
+      # end
+
+      # Fetch dates
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      def fetch_dates(doc)
+        dates = []
+        publish_date = doc.at("//span[@itemprop='releaseDate']").text
+        unless publish_date.empty?
+          dates << { type: 'published', on: publish_date }
+        end
+        dates
+      end
+
+      def fetch_contributors(code)
+        code.sub(/\s.*/, '').split('/').map do |abbrev|
+          case abbrev
+          when 'ISO'
+            name = 'International Organization for Standardization'
+            url = 'www.iso.org'
+          when 'IEC'
+            name = 'International Electrotechnical Commission'
+            url  = 'www.iec.ch'
+          end
+          { entity: { name: name, url: url, abbreviation: abbrev },
+            roles: ['publisher'] }
+        end
+      end
+
+      # Fetch ICS.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      def fetch_ics(doc)
+        doc.xpath('//th[contains(text(), "ICS")]/following-sibling::td/a').map do |i|
+          code = i.text.match(/[\d\.]+/).to_s.split '.'
+          { field: code[0], group: code[1], subgroup: code[2] }
+        end
+      end
+
+      # Fetch links.
+      # @param doc [Nokogiri::HTML::Document]
+      # @param url [String]
+      # @return [Array<Hash>]
+      def fetch_link(doc, url)
+        links = [{ type: 'src', content: url }]
+        obp_elms = doc.at_css('p.btn-preview a')
+        links << { type: 'obp', content: obp_elms[:href] } if obp_elms
+        links
+      end
+
+      # Fetch copyright.
+      # @param title [String]
+      # @return [Hash]
+      def fetch_copyright(code, doc)
+        owner_name = code.match(/.*?(?=\s)/).to_s
+        from = code.match(/(?<=:)\d{4}/).to_s
+        if from.empty?
+          from = doc.xpath("//span[@itemprop='releaseDate']").text
+            .match(/\d{4}/).to_s
+        end
+        { owner: { name: owner_name }, from: from }
+      end
+    end
+  end
+  # rubocop:enable Metrics/ModuleLength
+end

data/lib/iecbib/statuses.yml

@@ -0,0 +1,132 @@
+ACD:
+  status: Approved for CD
+  stage: '20.99'
+ACDV:
+  status: Approved for CDV
+  stage: '30.99'
+ADTR:
+  status: Approved for DTR
+  stage: '40.99'
+ADTS:
+  status: Approved for DTS
+  stage: '40.99'
+AFDIS:
+  status: Approved for FDIS
+  stage: '40.99'
+APUB:
+  status: Approved for publication
+  stage: '50.99'
+BPUB:
+  status: Being published
+  stage: '60.00'
+CAN:
+  status: Draft cancelled
+  stage: '20.98'
+CD:
+  status: Draft circulated as CD
+  stage: '30.00'
+CDM:
+  status: CD to be discussed at meeting
+  stage: '30.20'
+CCDV:
+  status: Draft circulated as CDV
+  stage: '40.00'
+CDVM:
+  status: Rejected CDV to be discussed at a meeting
+  stage: '40.93'
+CFDIS:
+  status: Draft circulated as FDIS
+  stage: '50.20'
+CDPAS:
+  status: Draft circulated as DPAS
+  stage: '50.20'
+CDTR:
+  status: Draft circulated as DTR
+  stage: '50.20'
+CDTS:
+  status: Draft circulated as DTS
+  stage: '50.20'
+DTRM:
+  status: Rejected DTR to be discussed at meeting
+  stage: '50.92'
+DTSM:
+  status: Rejected DTS to be discussed at meeting
+  stage: '50.92'
+DECFDIS:
+  status: FDIS at editing check
+  stage: '50.60'
+DECPUB:
+  status: Publication at editing check
+  stage: '60.00'
+DEL:
+  status: Deleted/abandoned
+  stage: '00.99'
+DELPUB:
+  status: Deleted publication
+  stage: '90.99'
+NCDV:
+  status: CDV rejected
+  stage: '40.98'
+NDTR:
+  status: DTR rejected
+  stage: '50.98'
+NDTS:
+  status: DTS rejected
+  stage: '50.98'
+NFDIS:
+  status: FDIS rejected
+  stage: '50.98'
+PCC:
+  status: Preparation of CC
+  stage: '30.92'
+PNW:
+  status: New work item proposal
+  stage: '10.00'
+PPUB:
+  status: Publication issued
+  stage: '60.60' # ?
+PRVC:
+  status: Preparation of RVC
+  stage: '40.92'
+PRVD:
+  status: Preparation of RVD
+  stage: '40.92'
+PRVDPAS:
+  status: Preparation of RVDPAS
+  stage: '40.92'
+PRVDTR:
+  status: Preparation of RVDTR
+  stage: '40.92'
+PRVDTS:
+  status: Preparation of RVDTS
+  stage: '40.92'
+PRVN:
+  status: Preparation of RVN
+  stage: '40.92'
+PWI:
+  status: Preliminary work item
+  stage: '00.00'
+RFDIS:
+  status: FDIS received and registered
+  stage: '50.00'
+RPUB:
+  status: Publication received and registered
+  stage: '60.60'
+TCDV:
+  status: Translation of CDV
+  stage: '50.00'
+TDTR:
+  status: Translation of DTR
+  stage: '50.00'
+TDTS:
+  status: Translation of DTS
+  stage: '50.00'
+TFDIS:
+  status: Translation of FDIS
+  stage: '50.00'
+TPUB:
+  status: Translation of publication
+  stage: '60.00'
+WPUB:
+  status: Publication withdrawn
+  stage: '95.99'

data/lib/iecbib/version.rb

@@ -0,0 +1,3 @@
+module Iecbib
+  VERSION = "0.1.1"
+end

data/lib/iecbib/workers_pool.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Workers poll.
+class WorkersPool
+  attr_accessor :nb_hits
+
+  def initialize(num_workers = 2)
+    @num_workers = num_workers < 2 ? 2 : num_workers
+    @queue = SizedQueue.new(num_workers * 2)
+    @result = []
+    @nb_hits = 0
+  end
+
+  def worker(&block)
+    @threads = Array.new @num_workers do
+      Thread.new do
+        until (item = @queue.pop) == :END
+          @result << yield(item) if block
+        end
+      end
+    end
+  end
+
+  def result
+    @threads.each(&:join)
+    @result
+  end
+
+  def <<(item)
+    @queue << item
+    self
+  end
+
+  def end
+    @num_workers.times { @queue << :END }
+  end
+
+  def size
+    @result.size
+  end
+end

data/lib/relaton/processor.rb

@@ -0,0 +1,19 @@
+require "relaton/processor"
+
+module Relaton
+  module Iecbib
+    class Processor < Relaton::Processor
+
+      def initialize
+        @short = :iecbib
+        @prefix = "IEC"
+        @defaultprefix = %r{^(IEC)[ /]|^IEV($| )}
+        @idtype = "IEC"
+      end
+
+      def get(code, date, opts)
+        ::Iecbib::IecBibliography.get(code, date, opts)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,193 @@
+--- !ruby/object:Gem::Specification
+name: iecbib
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Ribose Inc.
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-09-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: equivalent-xml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: iso-bib-item
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+description: 'IsoBib: retrieve ISO Standards for bibliographic use using the BibliographicItem
+  model'
+email:
+- open.source@ribose.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.adoc
+- Rakefile
+- bin/console
+- bin/setup
+- iecbib.gemspec
+- lib/iecbib.rb
+- lib/iecbib/hit.rb
+- lib/iecbib/hit_collection.rb
+- lib/iecbib/iec_bibliography.rb
+- lib/iecbib/scrapper.rb
+- lib/iecbib/statuses.yml
+- lib/iecbib/version.rb
+- lib/iecbib/workers_pool.rb
+- lib/relaton/processor.rb
+homepage: https://github.com/riboseinc/isobib
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: 'IsoBib: retrieve ISO Standards for bibliographic use using the BibliographicItem
+  model'
+test_files: []