relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/etsi/processor.rb

@@ -0,0 +1,69 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Etsi
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_etsi
+        @prefix = "ETSI"
+        @defaultprefix = %r{^ETSI\s}
+        @idtype = "ETSI"
+        @datasets = %w[etsi-csv]
+      end
+
+      # @param code [String]
+      # @param date [String, nil] year
+      # @param opts [Hash]
+      # @return [Relaton::Etsi::Item]
+      def get(code, date, opts)
+        require_relative "../etsi"
+        Bibliography.get(code, date, opts)
+      end
+
+      #
+      # Fetch all the documents from https://www.etsi.org
+      #
+      # @param [String] _source source name
+      # @param [Hash] opts
+      # @option opts [String] :output directory to output documents
+      # @option opts [String] :format output format (xml, yaml, bibxml)
+      #
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(**opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Etsi::Item]
+      def from_xml(xml)
+        require_relative "../etsi"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Etsi::Item]
+      def from_yaml(yaml)
+        require_relative "../etsi"
+        Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require "digest/md5" # not loaded by etsi.rb; needed on the cold path
+        require "relaton/version"
+        Digest::MD5.hexdigest Relaton::VERSION
+      end
+
+      #
+      # Remove index file
+      #
+      def remove_index_file
+        require_relative "../etsi"
+        Relaton::Index.find_or_create(:etsi, url: true, file: INDEX_FILE).remove_file
+      end
+    end
+  end
+end

data/lib/relaton/etsi/pubid.rb

@@ -0,0 +1,37 @@
+module Relaton
+  module Etsi
+    class PubId
+      class Parser
+        def initialize(id)
+          @strscan = StringScanner.new id
+        end
+
+        def parse
+          @strscan.scan(/^ETSI\s+/)
+          type = @strscan.scan(/\S+/)
+          @strscan.scan(/\s+/)
+          docnumber = @strscan.scan_until(/(?=\s(V\d+\.\d+\.\d+)|ed\.\d+)/)
+          version = @strscan.scan(/\d+\.\d+\.\d+/) if @strscan.scan(/\sV(?=\d+\.\d+\.\d+)/)
+          edition = @strscan.scan(/\d+/) if @strscan.scan(/\sed\.(?=\d+)/)
+          date = @strscan.scan(/\d{4}-\d{2}/) if @strscan.scan(/\s\(/)
+
+          { type: type, docnumber: docnumber, version: version, edition: edition, date: date }
+        end
+      end
+
+      attr_accessor :type, :docnumber, :version, :edition, :date
+
+      def initialize(type:, docnumber:, version:, edition:, date:)
+        @type = type
+        @docnumber = docnumber
+        @version = version
+        @edition = edition
+        @date = date
+      end
+
+      def self.parse(id)
+        new(**Parser.new(id).parse)
+      end
+    end
+  end
+end

data/lib/relaton/etsi/status.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Etsi
+    class Status < Lutaml::Model::Serializable
+      attribute :stage, :string, values: %W[
+        EN\sapproval SG\sapproval ES\sapproval Published Withdrawn Historical
+      ]
+
+      xml do
+        map_element "stage", to: :stage
+      end
+    end
+  end
+end

data/lib/relaton/etsi/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Etsi
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-etsi".freeze
+    end
+  end
+end

data/lib/relaton/etsi.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "mechanize"
+require "relaton/index"
+require_relative "version"
+# require_relative "relaton_etsi/pubid"
+require_relative "etsi/bibitem"
+require_relative "etsi/bibdata"
+require_relative "etsi/util"
+require_relative "etsi/bibliography"
+
+module Relaton
+  module Etsi
+    INDEX_FILE = "index-v1.yaml"
+
+    # Returns hash of gem versions used to generate data model.
+    # @return [String]
+    def grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+
+    extend self
+  end
+end

data/lib/relaton/gb/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Gb
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/gb/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Gb
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/gb/bibliography.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require_relative "hit_collection"
+
+# GB bib module.
+module Relaton
+  module Gb
+    # GB entry point class.
+    class Bibliography
+      class << self
+        # rubocop:disable Metrics/MethodLength
+        # @param text [Strin] code of standard for search
+        # @return [RelatonGb::HitCollection]
+        def search(text)
+          case text
+          when /^(GB|GJ|GS)/
+            # Scrape national standards.
+            Util.info "Fetching from openstd.samr.gov.cn ...", key: text
+            require_relative "gb_scraper"
+            GbScraper.scrape_page text
+          # when /^ZB/
+            # Scrape proffesional.
+          # when /^DB/
+            # Scrape local standard.
+          # when %r{^Q/}
+            # Enterprise standard
+          when %r{^T/[^\s]{2,6}\s}
+            # Scrape social standard.
+            Util.info "Fetching from www.ttbz.org.cn ...", key: text
+            require_relative "t_scraper"
+            TScraper.scrape_page text
+          else
+            # Scrape sector standard.
+            require "relaton/gb/sec_scraper"
+            SecScraper.scrape_page text
+          end
+        end
+        # rubocop:enable Metrics/MethodLength
+
+        # @param code [String] the GB standard Code to look up (e..g "GB/T 20223")
+        # @param year [String] the year the standard was published (optional)
+        # @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
+        # @return [String] Relaton XML serialisation of reference
+        def get(code, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+          if year.nil?
+            /^(?<code1>[^-]+)-(?<year1>[^-]+)$/ =~ code
+            unless code1.nil?
+              code = code1
+              year = year1
+            end
+          end
+
+          code += ".1" if opts[:all_parts]
+          code, year = code.split("-", 2) if code.include?("-")
+          ret = get1(code, year, opts)
+          return nil if ret.nil?
+
+          ret = ret.to_most_recent_reference unless year
+          ret = ret.to_all_parts if opts[:all_parts]
+          ret
+        end
+
+        private
+
+        def fetch_ref_err(code, year, missed_years) # rubocop:disable Metrics/MethodLength
+          # id = year ? "#{code}:#{year}" : code
+          # Util.info "WARNING: No match found on the GB website for `#{id}`. " \
+          #           "The code must be exactly like it is on the website."
+          unless missed_years.empty?
+            Util.info "(There was no match for `#{year}`, though there " \
+                      "were matches found for `#{missed_years.join('`, `')}`.)"
+          end
+          if /\d-\d/.match? code
+            Util.info "The provided document part may not exist, or " \
+                      "the document may no longer be published in parts."
+          else
+            Util.info "If you wanted to cite all document parts for the " \
+                      "reference, use `#{code} (all parts)`.\nIf the document " \
+                      "is not a standard, use its document type abbreviation " \
+                      "(TS, TR, PAS, Guide)."
+          end
+          nil
+        end
+
+        def get1(code, year, _opts)
+          # search must include year whenever available
+          searchcode = code + (year.nil? ? "" : "-#{year}")
+          result = search_filter(searchcode) || return
+          ret = results_filter(result, year)
+          if ret[:ret]
+            Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: searchcode
+            ret[:ret]
+          else
+            Util.info "Not found.", key: searchcode
+            fetch_ref_err(code, year, ret[:years])
+          end
+        end
+
+        def search_filter(code)
+          # search filter needs to incorporate year
+          docidrx = %r{^[^\s]+\s[\d.-]+}
+          result = search(code)
+          result.select! do |hit|
+            hit.docref && hit.docref.match(docidrx).to_s.include?(code)
+          end
+        end
+
+        # Sort through the results from Isobib, fetching them three at a time,
+        # and return the first result that matches the code,
+        # matches the year (if provided), and which # has a title (amendments do
+        # not).
+        # Only expects the first page of results to be populated.
+        # Does not match corrigenda etc (e.g. ISO 3166-1:2006/Cor 1:2007)
+        # If no match, returns any years which caused mismatch, for error
+        # reporting
+        def results_filter(result, year) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength
+          missed_years = []
+          result.each_slice(3) do |s| # ISO website only allows 3 connections
+            fetch_pages(s, 3).each do |r|
+              return { ret: r } if !year
+
+              r.date.select { |d| d.type == "published" }.each do |d|
+                return { ret: r } if year.to_i == d.at.to_date.year
+
+                missed_years << d.on(:year)
+              end
+            end
+          end
+          { years: missed_years }
+        end
+
+        # @param hits [RelatonBib::HitCollection<RelatonBib::Hit>]
+        # @param threads [Integer]
+        # @return [Array<RelatonBib::GbBibliographicItem>]
+        def fetch_pages(hits, threads)
+          workers = Core::WorkersPool.new threads
+          workers.worker { |w| { i: w[:i], hit: w[:hit].item } }
+          hits.each_with_index { |hit, i| workers << { i: i, hit: hit } }
+          workers.end
+          workers.result.sort_by { |x| x[:i] }.map { |x| x[:hit] }
+        end
+      end
+    end
+  end
+end

data/lib/relaton/gb/ccs.rb

@@ -0,0 +1,14 @@
+module Relaton
+  module Gb
+    class CCS < Bib::ICS
+      xml { root "ccs" }
+
+      def text
+        val = @text
+        return val if val && !val.empty?
+
+        Cnccs.fetch(code)&.description if code
+      end
+    end
+  end
+end

data/lib/relaton/gb/committee.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Gb
+    class Committee < Lutaml::Model::Serializable
+      attribute :type, :string, values: %w[technical provisional drafting]
+      attribute :content, :string
+
+      xml do
+        map_attribute "type", to: :type
+        map_content to: :content
+      end
+    end
+  end
+end

data/lib/relaton/gb/docidentifier.rb

@@ -0,0 +1,23 @@
+module Relaton
+  module Gb
+    class Docidentifier < Bib::Docidentifier
+      def content
+        @all_parts ? "#{super} (all parts)" : super
+      end
+
+      def to_all_parts!
+        remove_part!
+        remove_date!
+        @all_parts = true
+      end
+
+      def remove_part!
+        content.sub!(/\.\d+/, "")
+      end
+
+      def remove_date!
+        content.sub!(/-\d{4}$/, "")
+      end
+    end
+  end
+end

data/lib/relaton/gb/doctype.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Gb
+    class Doctype < Bib::Doctype
+      TYPES = %w[standard recommendation].freeze
+
+      attribute :type, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/gb/ext.rb

@@ -0,0 +1,36 @@
+require_relative "doctype"
+require_relative "committee"
+require_relative "structured_identifier"
+require_relative "stage_name"
+require_relative "gb_type"
+require_relative "ccs"
+
+module Relaton
+  module Gb
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :subdoctype, :string, values: %w[specification method-of-test vocabulary code-of-practice]
+      attribute :structuredidentifier, StructuredIdentifier
+      attribute :stagename, StageName
+      attribute :gbtype, GbType
+      attribute :ccs, CCS, collection: true
+      attribute :plannumber, :string
+
+      xml do
+        map_element "stagename", to: :stagename
+        map_element "gbtype", to: :gbtype
+        map_element "ccs", to: :ccs
+        map_element "plannumber", to: :plannumber
+      end
+
+      key_value do
+        map_element "stagename", to: :stagename
+        map_element "gbtype", to: :gbtype
+        map_element "ccs", to: :ccs
+        map_element "plannumber", to: :plannumber
+      end
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-gb"]
+    end
+  end
+end

data/lib/relaton/gb/gb_scraper.rb

@@ -0,0 +1,61 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+require "nokogiri"
+require_relative "scraper"
+
+module Relaton
+  module Gb
+    # National standard scraper.
+    module GbScraper
+      extend Scraper
+      SEARCH_URL = "https://openstd.samr.gov.cn/bzgk/gb/std_list"
+      DOC_URL = "http://openstd.samr.gov.cn/bzgk/gb/newGbInfo?hcno="
+
+      class << self
+        # @param text [Strin] code of standard for serarch
+        # @return [RelatonGb::HitCollection]
+        def scrape_page(text) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+          doc = agent.get("#{SEARCH_URL}?p.p2=#{CGI.escape(text)}")
+          hits = doc.xpath(
+            "//table[contains(@class, 'result_list')]/tbody[2]/tr",
+          ).map do |h|
+            ref = h.at "./td[2]/a"
+            pid = ref[:onclick].match(/[0-9A-F]+/).to_s
+            status = h.at("./td[7]").text.strip
+            rdate = h.at("./td[8]").text.strip
+            Hit.new pid: pid, docref: ref.text, scraper: self,
+                    release_date: rdate, status: status
+          end
+          HitCollection.new hits.sort_by(&:release_date).reverse
+        rescue Mechanize::Error => e
+          raise Relaton::RequestError, e.message
+        end
+
+        def agent
+          @agent ||= Mechanize.new
+        end
+
+        # @param hit [RelatonGb::Hit] standard's page id
+        # @return [RelatonGb::GbBibliographicItem]
+        def scrape_doc(hit)
+          src = DOC_URL + hit.pid
+          doc = agent.get src
+          ItemData.new(**scrapped_data(doc, src, hit))
+        rescue Mechanize::Error => e
+          raise Relaton::RequestError, e.message
+        end
+
+        # @param doc [Nokogiri::HTML]
+        # @param _ref [String]
+        # @return [Hash]
+        #   * :type [String]
+        #   * :name [String]
+        # def get_committee(doc, _ref)
+        #   name = doc.at("//div[contains(., '归口单位') or contains(., '归口部门')]/following-sibling::div")
+        #   Committee.new(type: "technical", content: name.text.delete("\r\n\t\t"))
+        # end
+      end
+    end
+  end
+end

data/lib/relaton/gb/gb_type.rb

@@ -0,0 +1,20 @@
+module Relaton
+  module Gb
+    class GbType < Lutaml::Model::Serializable
+      attribute :scope, :string, values: %w[national sector professional local enterprise social-group]
+      attribute :prefix, :string
+      attribute :mandate, :string, values: %w[mandatory recommended guidelines]
+      attribute :topic, :string, values: %w[
+        basic health-and-safety environment-protection engineering-and-construction
+        product method management-techniques other
+      ]
+
+      xml do
+        map_element "gbscope", to: :scope
+        map_element "gbprefix", to: :prefix
+        map_element "gbmandate", to: :mandate
+        map_element "gbtopic", to: :topic
+      end
+    end
+  end
+end

data/lib/relaton/gb/hit.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Gb
+    # Hit.
+    class Hit < Relaton::Core::Hit
+      # @return [String]
+      attr_reader :pid, :docref
+
+      # @return [Date, NilClass]
+      attr_reader :release_date
+
+      # @return [String, NilClass]
+      attr_reader :status
+
+      # @return [RelatonGb::GbScraper, RelatonGb::SecScraper, RelatonGb::TScraper]
+      attr_reader :scraper
+
+      # @param pid [String]
+      # @param docref [String]
+      # @parma scraper [RelatonGb::GbScraper, RelatonGb::SecScraper, RelatonGb::TScraper]
+      # @param release_date [String]
+      # @status [String, NilClass]
+      # @param hit_collection [RelatonGb:HitCollection, NilClass]
+      def initialize(pid:, docref:, scraper:, **args)
+        @pid            = pid
+        @docref         = docref
+        @scraper        = scraper
+        @release_date   = Date.parse args[:release_date] if args[:release_date]
+        @status         = args[:status]
+        @hit_collection = args[:hit_collection]
+      end
+
+      # Parse page.
+      # @return [Isobib::IsoBibliographicItem]
+      def item
+        @item ||= scraper.scrape_doc self
+      end
+
+      # @return [String]
+      def inspect
+        "<#{self.class}:#{format('%<id>#.14x', id: object_id << 1)} " \
+          "@fullIdentifier=\"#{@fetch&.shortref}\" " \
+          "@docref=\"#{docref}\">"
+      end
+    end
+  end
+end

data/lib/relaton/gb/hit_collection.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "hit"
+
+module Relaton
+  module Gb
+    # Page of hit collection
+    class HitCollection < Relaton::Core::HitCollection
+      # @param hits [Array<Hash>]
+      # @param hit_pages [Integer]
+      # @param scraper [RelatonGb::GbScraper, RelatonGb::SecScraper,
+      #   RelatonGb::TScraper]
+      def initialize(hits = [])
+        @array = hits
+        @fetched = false
+      end
+    end
+  end
+end

data/lib/relaton/gb/item.rb

@@ -0,0 +1,13 @@
+require_relative "docidentifier"
+require_relative "ext"
+
+module Relaton
+  module Gb
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/gb/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Gb
+    class ItemData < Relaton::Bib::ItemData
+    end
+  end
+end

data/lib/relaton/gb/processor.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "relaton/core/processor"
+
+module Relaton
+  module Gb
+    class Processor < Relaton::Core::Processor
+      def initialize
+        @short = :relaton_gb
+        @prefix = "CN"
+        @defaultprefix = %r{^(GB|GB/T|GB/Z) }
+        @idtype = "Chinese Standard"
+      end
+
+      # @param code [String]
+      # @param date [String, NilClass] year
+      # @param opts [Hash]
+      # @return [Relaton::Gb::ItemData, nil]
+      def get(code, date, opts)
+        require_relative "../gb"
+        Bibliography.get(code, date, opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Gb::ItemData]
+      def from_xml(xml)
+        require_relative "../gb"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Gb::ItemData]
+      def from_yaml(yaml)
+        require_relative "../gb" # defines Item — cache reads hit this cold
+        Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../gb"
+        @grammar_hash ||= ::Relaton::Gb.grammar_hash
+      end
+    end
+  end
+end

data/lib/relaton/gb/project_number.rb

@@ -0,0 +1,38 @@
+module Relaton
+  module Gb
+    class ProjectNumber < Lutaml::Model::Serializable
+      attribute :part, :integer
+      attribute :subpart, :integer
+      attribute :amendment, :integer
+      attribute :corrigendum, :integer
+      attribute :origyr, :string
+      attribute :content, :string
+
+      xml do
+        root "project-ntnumber"
+        map_attribute "part", to: :part
+        map_attribute "subpart", to: :subpart
+        map_attribute "amendment", to: :amendment
+        map_attribute "corrigendum", to: :corrigendum
+        map_attribute "origyr", to: :origyr
+        map_content to: :content
+      end
+
+      def to_all_parts!
+        remove_part!
+        remove_date!
+      end
+
+      def remove_part!
+        self.part = nil
+        self.subpart = nil
+        self.amendment = nil
+        self.corrigendum = nil
+      end
+
+      def remove_date!
+        self.origyr = nil
+      end
+    end
+  end
+end