relaton-cen 1.8.pre1

data/lib/relaton_cen/scrapper.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module RelatonCen
+  # Scrapper.
+  module Scrapper
+    COMMITTEES = {
+      "TC 459" =>
+        "ECISS - European Committee for Iron and Steel Standardization",
+    }.freeze
+
+    class << self
+      # Parse page.
+      # @param hit [RelatonCen::Hit]
+      # @return [RelatonBib::BibliographicItem]
+      def parse_page(hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        doc = hit.hit_collection.agent.get hit.hit[:url]
+        RelatonIsoBib::IsoBibliographicItem.new(
+          fetched: Date.today.to_s,
+          type: "standard",
+          docid: fetch_docid(hit.hit[:code]),
+          language: ["en"],
+          script: ["Latn"],
+          title: fetch_titles(doc),
+          doctype: "international-standard",
+          docstatus: fetch_status(doc),
+          ics: fetch_ics(doc),
+          date: fetch_dates(doc),
+          # contributor: fetch_contributors(doc),
+          editorialgroup: fetch_editorialgroup(doc),
+          structuredidentifier: fetch_structuredid(hit.hit),
+          abstract: fetch_abstract(doc),
+          copyright: fetch_copyright(doc),
+          link: fetch_link(doc.uri.to_s),
+          relation: fetch_relations(doc),
+          place: ["London"],
+        )
+      end
+
+      private
+
+      # @param doc [Mechanize::Page]
+      # @return [Array<RelatonIsobib::Ics>]
+      def fetch_ics(doc)
+        doc.xpath("//tr[th[.='ICS']]/td/text()").map do |ics|
+          RelatonIsoBib::Ics.new ics.text.match(/[^\s]+/).to_s
+        end
+      end
+
+      # Fetch abstracts.
+      # @param doc [Mechanize::Page]
+      # @return [Array<Hash>]
+      def fetch_abstract(doc)
+        content = doc.at("//tr[th[.='Abstract/Scope']]/td")
+        [{ content: content.text, language: "en", script: "Latn" }]
+      end
+
+      # Fetch docid.
+      # @param ref [String]
+      # @return [Array<RelatonBib::DocumentIdentifier>]
+      def fetch_docid(ref)
+        [RelatonBib::DocumentIdentifier.new(type: "CEN", id: ref)]
+      end
+
+      # Fetch status.
+      # @param doc [Mechanize::Page]
+      # @return [RelatonBib::DocumentStatus, NilClass]
+      def fetch_status(doc)
+        s = doc.at("//tr[th[.='Status']]/td")
+        return unless s
+
+        RelatonBib::DocumentStatus.new(stage: s.text.strip)
+      end
+
+      # Fetch workgroup.
+      # @param doc [Mechanize::Page]
+      # @return [RelatonIsoBib::EditorialGroup]
+      def fetch_editorialgroup(doc) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+        code = doc.at("//tr/td/h1/text()").text
+        title = doc.at("//tr/td/h1/span").text
+        %r{/(?<type>\w+)(?:\s(?<num>[^/]+))?$} =~ code
+        tc = []
+        COMMITTEES.each do |k, v|
+          next unless code.include? k
+
+          t, n = k.split " "
+          tc << RelatonIsoBib::IsoSubgroup.new(name: v, type: t, number: n)
+        end
+        sc = []
+        if tc.any?
+          sc << RelatonIsoBib::IsoSubgroup.new(name: title, type: type, number: num)
+        else
+          tc << RelatonIsoBib::IsoSubgroup.new(name: title, type: type, number: num)
+        end
+        RelatonIsoBib::EditorialGroup.new(technical_committee: tc,
+                                          subcommittee: sc)
+      end
+
+      # @param hit [RelatonCen::Hit]
+      # @return [RelatonIsoBib::StructuredIdentifier]
+      def fetch_structuredid(hit)
+        %r{(?<pnum>\d+)(?:-(?<part>\d+))?(?:-(?<subpart>\d+))?} =~ hit[:code]
+        RelatonIsoBib::StructuredIdentifier.new(
+          project_number: pnum, part: part, subpart: subpart, type: "CEN",
+        )
+      end
+
+      # Fetch relations.
+      # @param doc [Mechanize::Page]
+      # @return [Array<Hash>]
+      def fetch_relations(doc) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+        doc.xpath(
+          "//div[@id='DASHBOARD_LISTRELATIONS']/table/tr[th[.!='Sales Points']]",
+        ).each_with_object([]) do |rt, a|
+          t = rt.at("th").text
+          type = case t
+                 when "Supersedes" then "obsoletes"
+                 when /Normative reference/ then "cites"
+                 else t.downcase
+                 end
+          rt.xpath("td/a").each do |r|
+            fref = RelatonBib::FormattedRef.new(content: r.text, language: "en",
+                                                script: "Latn")
+            link = fetch_link HitCollection::DOMAIN + r[:href]
+            bibitem = RelatonBib::BibliographicItem.new(
+              formattedref: fref, type: "standard", link: link,
+            )
+            a << { type: type, bibitem: bibitem }
+          end
+        end
+      end
+
+      # Fetch titles.
+      # @param doc [Mechanize::Page]
+      # @return [RelatonBib::TypedTitleStringCollection]
+      def fetch_titles(doc)
+        te = doc.at("//tr[th[.='Title']]/td").text.strip
+        RelatonBib::TypedTitleString.from_string te, "en", "Latn"
+      end
+
+      # Fetch dates
+      # @param hit [Mechanize::Page]
+      # @return [Array<Hash>]
+      def fetch_dates(doc) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength
+        doc.xpath("//div[@id='DASHBOARD_LISTIMPLEMENTATIONDATES']/table/tr")
+          .each_with_object([]) do |d, a|
+          on = d.at("td").text
+          next if on.empty?
+
+          t = d.at("th").text
+          type = case t
+                 when /DOR/ then "adapted"
+                 when /DAV/ then "issued"
+                 when /DOA/ then "announced"
+                 when /DOP/ then "published"
+                 when /DOW/ then "obsoleted"
+                 else t.downcase
+                 end
+          a << { type: type, on: on }
+        end
+      end
+
+      # Fetch contributors
+      # @param doc [Mechanize::Page]
+      # @return [Array<Hash>]
+      # def fetch_contributors(doc)
+      #   contrib = { role: [type: "publisher"] }
+      #   contrib[:entity] = owner_entity doc
+      #   [contrib]
+      # end
+
+      # Fetch links.
+      # @param url [String]
+      # @return [Array<Hash>]
+      def fetch_link(url)
+        [{ type: "src", content: url }]
+      end
+
+      # Fetch copyright.
+      # @param doc [Mechanize::Page]
+      # @return [Array<Hash>]
+      def fetch_copyright(doc)
+        date = doc.at("//tr[th[.='date of Availability (DAV)']]/td").text
+        owner = owner_entity
+        from = date.match(/^\d{4}/).to_s
+        [{ owner: [owner], from: from }]
+      end
+
+      # @return [Hash]
+      def owner_entity
+        {
+          abbreviation: "CEN",
+          name: "European Committee for Standardization",
+          url: "https://cen.eu",
+        }
+      end
+    end
+  end
+end

data/lib/relaton_cen/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module RelatonCen
+  VERSION = "1.8.pre1"
+end

data/lib/relaton_cen/xml_parser.rb

@@ -0,0 +1,28 @@
+require "nokogiri"
+
+module RelatonCen
+  class XMLParser < RelatonIsoBib::XMLParser
+    class << self
+      private
+
+      # Override RelatonBib::XMLParser#item_data method.
+      # @param isoitem [Nokogiri::XML::Element]
+      # @returtn [Hash]
+      # def item_data(isoitem)
+      #   data = super
+      #   ext = isoitem.at "./ext"
+      #   return data unless ext
+
+      #   data[:price_code] = ext.at("./price-code")&.text
+      #   data[:cen_processing] = ext.at("./cen-processing")&.text
+      #   data
+      # end
+
+      # @param item_hash [Hash]
+      # @return [RelatonBib::BibliographicItem]
+      # def bib_item(item_hash)
+      #   RelatonIsoBib::IsoBibliographicItem.new **item_hash
+      # end
+    end
+  end
+end

data/relaton_cen.gemspec

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "lib/relaton_cen/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "relaton-cen"
+  spec.version       = RelatonCen::VERSION
+  spec.authors       = ["Ribose Inc."]
+  spec.email         = ["open.source@ribose.com"]
+
+  spec.summary       = "RelatonItu: retrieve Cenelec Standards for bibliographic use "\
+                       "using the BibliographicItem model"
+  spec.description   = "RelatonItu: retrieve Cenelec Standards for bibliographic use "\
+                       "using the BibliographicItem model"
+  spec.homepage      = "https://github.com/metanorma/relaton-cen"
+  spec.license       = "BSD-2-Clause"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+
+  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "equivalent-xml", "~> 0.6"
+  spec.add_development_dependency "pry-byebug"
+  spec.add_development_dependency "ruby-jing"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "vcr", "~> 5.0.0"
+  spec.add_development_dependency "webmock"
+
+  spec.add_dependency "mechanize"
+  spec.add_dependency "relaton-iso-bib", "~> 1.8.0"
+
+  # For more information and examples about making a new gem, checkout our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

metadata

@@ -0,0 +1,185 @@
+--- !ruby/object:Gem::Specification
+name: relaton-cen
+version: !ruby/object:Gem::Version
+  version: 1.8.pre1
+platform: ruby
+authors:
+- Ribose Inc.
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2021-06-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: equivalent-xml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-jing
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: relaton-iso-bib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+description: 'RelatonItu: retrieve Cenelec Standards for bibliographic use using the
+  BibliographicItem model'
+email:
+- open.source@ribose.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/rake.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- Gemfile
+- LICENSE.txt
+- README.adoc
+- Rakefile
+- bin/console
+- bin/rspec
+- bin/setup
+- grammars/basicdoc.rng
+- grammars/biblio.rng
+- grammars/isodoc.rng
+- grammars/isostandard.rng
+- grammars/reqt.rng
+- lib/relaton_cen.rb
+- lib/relaton_cen/cen_bibliography.rb
+- lib/relaton_cen/committees.yaml
+- lib/relaton_cen/hit.rb
+- lib/relaton_cen/hit_collection.rb
+- lib/relaton_cen/processor.rb
+- lib/relaton_cen/scrapper.rb
+- lib/relaton_cen/version.rb
+- lib/relaton_cen/xml_parser.rb
+- relaton_cen.gemspec
+homepage: https://github.com/metanorma/relaton-cen
+licenses:
+- BSD-2-Clause
+metadata:
+  homepage_uri: https://github.com/metanorma/relaton-cen
+  source_code_uri: https://github.com/metanorma/relaton-cen
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: 'RelatonItu: retrieve Cenelec Standards for bibliographic use using the BibliographicItem
+  model'
+test_files: []