relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/cen/hit_collection.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Cen
+    # Page of hit collection.
+    class HitCollection < Relaton::Core::HitCollection
+      DOMAIN = "https://standards.cencenelec.eu"
+
+      # @param ref [String]
+      # @param year [String]
+      # def initialize(ref, year = nil) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+      #   super ref, year
+      def search
+        if !ref || ref.empty?
+          @array = []
+          return self
+        end
+
+        redirect_page = agent.get DOMAIN
+        redirect_url = redirect_page.body.slice(/(?<=follow the <a href=')#{DOMAIN}[^']+/)
+        search_page = agent.get redirect_url
+        form = search_page.form_with(id: "wwvFlowForm")
+        ref_field = form.field_with(id: "STAND_REF")
+        ref_field.value = ref
+        resp = agent.submit form
+        @array = hits resp
+        sort
+      end
+
+      def agent
+        @agent ||= Mechanize.new.tap { |a| a.user_agent_alias = "Mac Safari" }
+      end
+
+      def select!(&block)
+        @array.select!(&block)
+        self
+      end
+
+      private
+
+      def sort
+        @array.sort! do |a, b|
+          ap = Bibliography.code_to_parts a.hit[:code]
+          bp = Bibliography.code_to_parts b.hit[:code]
+          s = ap[:code] <=> bp[:code]
+          s = ap[:part].to_s <=> bp[:part].to_s if s.zero?
+          s = bp[:year].to_s <=> ap[:year].to_s if s.zero?
+          s = ap[:amd].to_s <=> bp[:amd].to_s if s.zero?
+          s = ap[:amy].to_s <=> bp[:amy].to_s if s.zero?
+          s = ap[:ac].to_s <=> bp[:ac].to_s if s.zero?
+          s
+        end
+        self
+      end
+
+      # @param resp [Mechanize::Page]
+      # @return [Array<RelatonCen::Hit>]
+      def hits(resp)
+        resp.xpath("//table[@class='dashlist']/tbody/tr/td[2]").map do |h|
+          ref = h.at("strong/a")
+          code = ref.text.strip
+          url = ref[:href]
+          Hit.new({ code: code, url: url }, self)
+        end
+      end
+    end
+  end
+end

data/lib/relaton/cen/item_data.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Cen
+    # Item data class.
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/cen/model/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Cen
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/cen/model/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Cen
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/cen/model/docidentifier.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Cen
+    class Docidentifier < Bib::Docidentifier
+      def remove_date!
+        self.content.sub!(/:\d{4}$/, "")
+      end
+    end
+  end
+end

data/lib/relaton/cen/model/ext.rb

@@ -0,0 +1,11 @@
+require_relative "structured_identifier"
+
+module Relaton
+  module Cen
+    class Ext < Bib::Ext
+      attribute :structuredidentifier, StructuredIdentifier, collection: true, initialize_empty: true
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-cen"]
+    end
+  end
+end

data/lib/relaton/cen/model/item.rb

@@ -0,0 +1,14 @@
+require_relative "../item_data"
+require_relative "docidentifier"
+require_relative "ext"
+
+module Relaton
+  module Cen
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/cen/model/structured_identifier.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Cen
+    class StructuredIdentifier < Bib::StructuredIdentifier
+      def remove_date!
+        self.year = nil
+      end
+    end
+  end
+end

data/lib/relaton/cen/processor.rb

@@ -0,0 +1,40 @@
+require_relative "../cen"
+
+module Relaton
+  module Cen
+    class Processor < Relaton::Core::Processor
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_cen
+        @prefix = "CEN"
+        @defaultprefix = %r{^(C?EN|ENV|CWA|HD|CR)[\s/]}
+        @idtype = "CEN"
+      end
+
+      # @param code [String]
+      # @param date [String, NilClass] year
+      # @param opts [Hash]
+      # @return [RelatonBib::BibliographicItem]
+      def get(code, date, opts)
+        Bibliography.get(code, date, opts)
+      end
+
+      # @param xml [String]
+      # @return [RelatonBib::BibliographicItem]
+      def from_xml(xml)
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [RelatonBib::BibliographicItem]
+      def from_yaml(yaml)
+        Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        @grammar_hash ||= Cen.grammar_hash
+      end
+    end
+  end
+end

data/lib/relaton/cen/scraper.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Cen
+    # Scraper.
+    module Scraper
+      COMMITTEES = {
+        "TC 459" =>
+          "ECISS - European Committee for Iron and Steel Standardization",
+      }.freeze
+
+      class << self
+        # Parse page.
+        # @param hit [RelatonCen::Hit]
+        # @return [RelatonIsoBib::IsoBibliographicItem]
+        def parse_page(hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+          doc = hit.hit_collection.agent.get hit.hit[:url]
+          ItemData.new(
+            fetched: Date.today.to_s,
+            type: "standard",
+            docidentifier: fetch_docid(hit.hit[:code]),
+            language: ["en"],
+            script: ["Latn"],
+            title: fetch_titles(doc),
+            status: fetch_status(doc),
+            date: fetch_dates(doc),
+            contributor: fetch_contributors(doc),
+            # editorialgroup: fetch_editorialgroup(doc),
+            abstract: fetch_abstract(doc),
+            copyright: fetch_copyright(doc),
+            source: fetch_source(doc.uri.to_s),
+            relation: fetch_relations(doc),
+            place: [Bib::Place.new(city: "London")],
+            ext: fetch_ext(doc, hit)
+          )
+        end
+
+        private
+
+        # @param doc [Mechanize::Page]
+        # @return [Array<Relaton::Bib::ICS>]
+        def fetch_ics(doc)
+          doc.xpath("//tr[th[.='ICS']]/td/text()").filter_map do |ics|
+            ics_code = ics.text.match(/[^\s]+/).to_s.gsub("\u00A0", "")
+            next if ics_code.empty?
+
+            isoics = Isoics.fetch(ics_code)
+            Bib::ICS.new code: ics_code, text: isoics.description
+          end
+        end
+
+        # Fetch abstracts.
+        # @param doc [Mechanize::Page]
+        # @return [Array<Relaton::Bib::LocalizedMarkedUpString>]
+        def fetch_abstract(doc)
+          content = doc.at("//tr[th[.='Abstract/Scope']]/td")
+          [Bib::Abstract.new(content: content.text, language: "en", script: "Latn")]
+        end
+
+        # Fetch docid.
+        # @param ref [String]
+        # @return [Array<Relaton::Cen::Docidentifier>]
+        def fetch_docid(ref)
+          [Docidentifier.new(type: "CEN", content: ref, primary: true)]
+        end
+
+        # Fetch status.
+        # @param doc [Mechanize::Page]
+        # @return [Relaton::Bib::DocumentStatus, nil]
+        def fetch_status(doc)
+          s = doc.at("//tr[th[.='Status']]/td")
+          return unless s
+
+          stage = Bib::Status::Stage.new(content: s.text.strip)
+          Bib::Status.new(stage: stage)
+        end
+
+        # @param hit [RelatonCen::Hit]
+        # @return [Relaton::Bib::StructuredIdentifier]
+        def fetch_structuredid(hit)
+          %r{(?<docnum>\d+)(?:-(?<part>\d+))?(?:-(?<subpart>\d+))?} =~ hit[:code]
+          partnumber = [part, subpart].compact.join(":")
+          StructuredIdentifier.new(docnumber: docnum, partnumber: partnumber, agency: ["CEN"])
+        end
+
+        # Fetch relations.
+        # @param doc [Mechanize::Page]
+        # @return [Array<Relaton::Bib::Relation>]
+        def fetch_relations(doc)
+          doc.xpath(
+            "//div[@id='DASHBOARD_LISTRELATIONS']/table/tr[th[.!='Sales Points']]",
+          ).each_with_object([]) do |rt, acc|
+            type = relation_type rt.at("th").text.downcase
+            rt.xpath("td/a").each do |r|
+              acc << Bib::Relation.new(type: type, bibitem: create_relation(r))
+            end
+          end
+        end
+
+        def relation_type(type)
+          case type
+          when "supersedes" then "obsoletes"
+          when "superseded by" then "obsoletedBy"
+          when /bibliographic references/ then "cites"
+          when /normative reference/ then "cites"
+          else type
+          end
+        end
+
+        def create_relation(rel)
+          source = fetch_source HitCollection::DOMAIN + rel[:href]
+          ItemData.new(formattedref: Bib::Formattedref.new(content: rel.text), type: "standard", source: source)
+        end
+
+        # Fetch titles.
+        # @param doc [Mechanize::Page]
+        # @return [Array<Relaton::Bib::Title>]
+        def fetch_titles(doc)
+          te = doc.at("//tr[th[.='Title']]/td").text.strip
+          Bib::Title.from_string te, "en", "Latn"
+        end
+
+        # Fetch dates
+        # @param hit [Mechanize::Page]
+        # @return [Array<Relaton::Bib::Date>]
+        def fetch_dates(doc) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength
+          doc.xpath("//div[@id='DASHBOARD_LISTIMPLEMENTATIONDATES']/table/tr")
+            .each_with_object([]) do |d, acc|
+            at = d.at("td").text
+            next if at.empty?
+
+            t = d.at("th").text
+            type = case t
+                  when /DOR/ then "adapted"
+                  when /DAV/ then "issued"
+                  when /DOA/ then "announced"
+                  when /DOP/ then "published"
+                  when /DOW/ then "obsoleted"
+                  else t.downcase
+                  end
+            acc << Bib::Date.new(type: type, at: at)
+          end
+        end
+
+        # Fetch contributors
+        # @param doc [Mechanize::Page]
+        # @return [Array<Relaton::Bib::Contributor>]
+        def fetch_contributors(doc)
+          code = doc.at("//tr/td/h1/text()").text
+          title = doc.at("//tr/td[3]/h1").text
+          %r{/(?<type>\w+)(?:\s(?<num>[^/]+))?$} =~ code
+          org = owner_entity
+
+          COMMITTEES.each do |k, name|
+            next unless code.include? k
+
+            wg_type, number = k.split
+            org.subdivision << create_subdivision("technical-committee", name, wg_type, number)
+          end
+
+          subdiv_type = org.subdivision.any? ? "subcommittee" : "technical-committee"
+          org.subdivision << create_subdivision(subdiv_type, title, type, num)
+          role_desc = Bib::LocalizedMarkedUpString.new(content: "committee")
+          role = Bib::Contributor::Role.new(type: "author", description: [role_desc])
+          [Bib::Contributor.new(role: [role], organization: org)]
+        end
+
+        def create_subdivision(type, name, subtype, wg_num = nil)
+          subdiv_name = Bib::TypedLocalizedString.new(content: name)
+          id = Array(wg_num).map { |wn| Bib::OrganizationType::Identifier.new(type: "number", content: wg_num) }
+          Bib::Subdivision.new( type: type, subtype: subtype, name: [subdiv_name], identifier: id)
+        end
+
+        # Fetch links.
+        # @param url [String]
+        # @return [Array<Relaton::Bib::Uri>]
+        def fetch_source(url)
+          [Bib::Uri.new(type: "src", content: url)]
+        end
+
+        # Fetch copyright.
+        # @param doc [Mechanize::Page]
+        # @return [Array<Bib::Copyright>]
+        def fetch_copyright(doc)
+          date = doc.at("//tr[th[.='date of Availability (DAV)']]/td").text
+          owner = Bib::ContributionInfo.new(organization: owner_entity)
+          from = date.match(/^\d{4}/).to_s
+          [Bib::Copyright.new(owner: [owner], from: from)]
+        end
+
+        # @return [Hash]
+        def owner_entity
+          org_name = Bib::TypedLocalizedString.new(content: "European Committee for Standardization")
+          org_abbr = Bib::TypedLocalizedString.new(content: "CEN")
+          uri = Bib::Uri.new(content: "https://cen.eu")
+          Bib::Organization.new(name: [org_name], abbreviation: org_abbr, uri: [uri])
+        end
+
+        def fetch_ext(doc, hit)
+          Ext.new(
+            doctype: Bib::Doctype.new(content: "international-standard"),
+            flavor: "cen",
+            ics: fetch_ics(doc),
+            structuredidentifier: fetch_structuredid(hit.hit),
+          )
+        end
+      end
+    end
+  end
+end

data/lib/relaton/cen/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Cen
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-cen".freeze
+    end
+  end
+end

data/lib/relaton/cen.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "digest/md5"
+require "mechanize"
+require "relaton/core"
+require "isoics"
+require "relaton/bib"
+require_relative "version"
+require_relative "cen/util"
+require_relative "cen/model/item"
+require_relative "cen/model/bibitem"
+require_relative "cen/model/bibdata"
+require_relative "cen/scraper"
+require_relative "cen/hit_collection"
+require_relative "cen/hit"
+require_relative "cen/bibliography"
+
+module Relaton
+  module Cen
+    # Returns hash of XML greammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/cie/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Cie
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/cie/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Cie
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/cie/bibliography.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal:true
+
+module Relaton
+  module Cie
+  # IETF bibliography module
+    module Bibliography
+      class << self
+        # @param code [String] the ECMA standard Code to look up (e..g "ECMA-6")
+        # @return [Relaton::Cie::ItemData]
+        def search(code)
+          Scrapper.scrape_page code
+        end
+
+        # @param code [String] the ECMA standard Code to look up (e..g "ECMA-6")
+        # @param year [String] not used
+        # @param opts [Hash] not used
+        # @return [Relaton::Cie::ItemData] Relaton of reference
+        def get(code, _year = nil, _opts = {})
+          Util.info "Fetching from Relaton repository ...", key: code
+          result = search code
+          if result
+            Util.info "Found: `#{result.docidentifier.first.content}`", key: code
+          else
+            Util.info "Not found.", key: code
+          end
+          result
+        end
+      end
+    end
+  end
+end