relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/etsi/doctype.rb

@@ -0,0 +1,30 @@
+module Relaton
+  module Etsi
+    class Doctype < Bib::Doctype
+      DOCTYPES = {
+        "EN" => "European Standard",
+        "ES" => "ETSI Standard",
+        "EG" => "ETSI Guide",
+        "TS" => "Technical Specification",
+        "GS" => "Group Specification",
+        "GR" => "Group Report",
+        "TR" => "Technical Report",
+        "ETR" => "ETSI Technical Report",
+        "GTS" => "GSM Technical Specification",
+        "SR" => "Special Report",
+        "TCRTR" => "Technical Committee Reference Technical Report",
+        "TBR" => "Technical Basis for Regulation",
+        "ETS" => "European Telecommunication Standard",
+        "I-ETS" => "Interim European Telecommunication Standard",
+        "NET" => "Norme Européenne de Télécommunication",
+      }.freeze
+
+      attribute :abbreviation, :string, values: DOCTYPES.keys
+      attribute :content, :string, values: DOCTYPES.values
+
+      def self.create_from_abbreviation(abbreviation)
+        new(content: DOCTYPES[abbreviation], abbreviation: abbreviation)
+      end
+    end
+  end
+end

data/lib/relaton/etsi/ext.rb

@@ -0,0 +1,31 @@
+require_relative "doctype"
+
+module Relaton
+  module Etsi
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :marker, :string, values: %w[Current Superseded]
+      attribute :frequency, :string, collection: true
+      attribute :mandate, :string, collection: true
+      attribute :custom_collection, :string, values: %W[
+        HSs\scited\sin\sOJ HSs\snot\syet\scited\sin\sOJ HSs\sRED\scited\sin\sOJ HSs\sEMC\scited\sin\sOJ
+      ]
+
+      xml do
+        map_element "marker", to: :marker
+        map_element "frequency", to: :frequency
+        map_element "mandate", to: :mandate
+        map_element "custom-collection", to: :custom_collection
+      end
+
+      key_value do
+        map_element "marker", to: :marker
+        map_element "frequency", to: :frequency
+        map_element "mandate", to: :mandate
+        map_element "custom_collection", to: :custom_collection
+      end
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-etsi"]
+    end
+  end
+end

data/lib/relaton/etsi/item.rb

@@ -0,0 +1,15 @@
+require "relaton/bib"
+require_relative "item_data"
+require_relative "ext"
+require_relative "status"
+
+module Relaton
+  module Etsi
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+      attribute :status, Status
+    end
+  end
+end

data/lib/relaton/etsi/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Etsi
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/etsi/processor.rb

@@ -0,0 +1,71 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Etsi
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_etsi
+        @prefix = "ETSI"
+        @defaultprefix = %r{^ETSI\s}
+        @idtype = "ETSI"
+        @datasets = %w[etsi-csv]
+      end
+
+      # @param code [String]
+      # @param date [String, nil] year
+      # @param opts [Hash]
+      # @return [Relaton::Etsi::Item]
+      def get(code, date, opts)
+        require_relative "../etsi"
+        Bibliography.get(code, date, opts)
+      end
+
+      #
+      # Fetch all the documents from https://www.etsi.org
+      #
+      # @param [String] _source source name
+      # @param [Hash] opts
+      # @option opts [String] :output directory to output documents
+      # @option opts [String] :format output format (xml, yaml, bibxml)
+      #
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(**opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Etsi::Item]
+      def from_xml(xml)
+        require_relative "../etsi"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Etsi::Item]
+      def from_yaml(yaml)
+        require_relative "../etsi"
+        Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require "digest/md5" # not loaded by etsi.rb; needed on the cold path
+        require "relaton/version"
+        Digest::MD5.hexdigest Relaton::VERSION
+      end
+
+      #
+      # Remove index file
+      #
+      def remove_index_file
+        require_relative "../etsi"
+        Relaton::Index.find_or_create(
+          :etsi, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end

data/lib/relaton/etsi/pubid.rb

@@ -0,0 +1,37 @@
+module Relaton
+  module Etsi
+    class PubId
+      class Parser
+        def initialize(id)
+          @strscan = StringScanner.new id
+        end
+
+        def parse
+          @strscan.scan(/^ETSI\s+/)
+          type = @strscan.scan(/\S+/)
+          @strscan.scan(/\s+/)
+          docnumber = @strscan.scan_until(/(?=\s(V\d+\.\d+\.\d+)|ed\.\d+)/)
+          version = @strscan.scan(/\d+\.\d+\.\d+/) if @strscan.scan(/\sV(?=\d+\.\d+\.\d+)/)
+          edition = @strscan.scan(/\d+/) if @strscan.scan(/\sed\.(?=\d+)/)
+          date = @strscan.scan(/\d{4}-\d{2}/) if @strscan.scan(/\s\(/)
+
+          { type: type, docnumber: docnumber, version: version, edition: edition, date: date }
+        end
+      end
+
+      attr_accessor :type, :docnumber, :version, :edition, :date
+
+      def initialize(type:, docnumber:, version:, edition:, date:)
+        @type = type
+        @docnumber = docnumber
+        @version = version
+        @edition = edition
+        @date = date
+      end
+
+      def self.parse(id)
+        new(**Parser.new(id).parse)
+      end
+    end
+  end
+end

data/lib/relaton/etsi/status.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Etsi
+    class Status < Lutaml::Model::Serializable
+      attribute :stage, :string, values: %W[
+        EN\sapproval SG\sapproval ES\sapproval Published Withdrawn Historical
+      ]
+
+      xml do
+        map_element "stage", to: :stage
+      end
+    end
+  end
+end

data/lib/relaton/etsi/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Etsi
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-etsi".freeze
+    end
+  end
+end

data/lib/relaton/etsi.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "mechanize"
+require "pubid"
+require "relaton/index"
+require_relative "version"
+# require_relative "relaton_etsi/pubid"
+require_relative "etsi/bibitem"
+require_relative "etsi/bibdata"
+require_relative "etsi/util"
+require_relative "etsi/bibliography"
+
+module Relaton
+  module Etsi
+    INDEXFILE = "index-v2".freeze
+
+    # Returns hash of gem versions used to generate data model.
+    # @return [String]
+    def grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+
+    extend self
+  end
+end

data/lib/relaton/gb/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Gb
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/gb/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Gb
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/gb/bibliography.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require_relative "hit_collection"
+
+# GB bib module.
+module Relaton
+  module Gb
+    # GB entry point class.
+    class Bibliography
+      class << self
+        # rubocop:disable Metrics/MethodLength
+        # @param text [Strin] code of standard for search
+        # @return [RelatonGb::HitCollection]
+        def search(text)
+          case text
+          when /^(GB|GJ|GS)/
+            # Scrape national standards.
+            Util.info "Fetching from openstd.samr.gov.cn ...", key: text
+            require_relative "gb_scraper"
+            GbScraper.scrape_page text
+          # when /^ZB/
+            # Scrape proffesional.
+          # when /^DB/
+            # Scrape local standard.
+          # when %r{^Q/}
+            # Enterprise standard
+          when %r{^T/[^\s]{2,6}\s}
+            # Scrape social standard.
+            Util.info "Fetching from www.ttbz.org.cn ...", key: text
+            require_relative "t_scraper"
+            TScraper.scrape_page text
+          else
+            # Scrape sector standard.
+            require "relaton/gb/sec_scraper"
+            SecScraper.scrape_page text
+          end
+        end
+        # rubocop:enable Metrics/MethodLength
+
+        # @param code [String] the GB standard Code to look up (e..g "GB/T 20223")
+        # @param year [String] the year the standard was published (optional)
+        # @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
+        # @return [Relaton::Gb::ItemData, nil]
+        # @raise [Pubid::Errors::ParseError] when the code is not a GB
+        #   identifier
+        def get(code, year = nil, opts = {})
+          require "pubid"
+          pubid = ::Pubid::Gb::Identifier.parse(code)
+          year = (year || pubid.year)&.to_s
+          pubid = pubid.exclude(:year)
+          pubid.part = "1" if opts[:all_parts]
+          ret = get1(pubid, year, opts)
+          return nil if ret.nil?
+
+          ret = ret.to_most_recent_reference unless year
+          ret = ret.to_all_parts if opts[:all_parts]
+          ret
+        end
+
+        private
+
+        # @param pubid [Pubid::Gb::Identifier] the reference without a year
+        def fetch_ref_err(pubid, year, missed_years) # rubocop:disable Metrics/MethodLength
+          unless missed_years.empty?
+            Util.info "(There was no match for `#{year}`, though there " \
+                      "were matches found for `#{missed_years.join('`, `')}`.)"
+          end
+          if pubid.part
+            Util.info "The provided document part may not exist, or " \
+                      "the document may no longer be published in parts."
+          else
+            Util.info "If you wanted to cite all document parts for the " \
+                      "reference, use `#{pubid} (all parts)`.\nIf the document " \
+                      "is not a standard, use its document type abbreviation " \
+                      "(TS, TR, PAS, Guide)."
+          end
+          nil
+        end
+
+        # @param pubid [Pubid::Gb::Identifier] the reference without a year
+        # @param year [String, nil]
+        def get1(pubid, year, _opts)
+          # search must include year whenever available
+          query = with_year(pubid, year)
+          searchcode = query.to_s
+          result = search_filter(query) || return
+          ret = results_filter(result, year)
+          if ret[:ret]
+            Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: searchcode
+            ret[:ret]
+          else
+            Util.info "Not found.", key: searchcode
+            fetch_ref_err(pubid, year, ret[:years])
+          end
+        end
+
+        # @param pubid [Pubid::Gb::Identifier]
+        # @param year [String, nil]
+        # @return [Pubid::Gb::Identifier] a copy of pubid that carries the year
+        def with_year(pubid, year)
+          return pubid unless year
+
+          dated = pubid.exclude
+          dated.date = ::Pubid::Components::Date.new(year: year)
+          dated
+        end
+
+        # Keeps the hits that name the queried document. A query with no year
+        # matches every year. A query with no part matches only the part-less
+        # document: in GB, `GB/T 20223` and `GB/T 20223.1` are two documents.
+        #
+        # @param query [Pubid::Gb::Identifier]
+        # @return [Relaton::Gb::HitCollection, nil]
+        def search_filter(query)
+          ignore = query.year ? [] : %i[year]
+          result = search(query.to_s)
+          result&.select! do |hit|
+            hit_pubid = parse_docref(hit.docref)
+            hit_pubid && query.matches?(hit_pubid, ignore: ignore)
+          end
+        end
+
+        # A portal docref is data, so a value pubid cannot read is nil and
+        # drops the hit.
+        #
+        # @param docref [String, nil]
+        # @return [Pubid::Gb::Identifier, nil]
+        def parse_docref(docref)
+          ::Pubid::Gb::Identifier.parse(docref) if docref
+        rescue StandardError
+          nil
+        end
+
+        # Sort through the results from Isobib, fetching them three at a time,
+        # and return the first result that matches the code,
+        # matches the year (if provided), and which # has a title (amendments do
+        # not).
+        # Only expects the first page of results to be populated.
+        # Does not match corrigenda etc (e.g. ISO 3166-1:2006/Cor 1:2007)
+        # If no match, returns any years which caused mismatch, for error
+        # reporting
+        def results_filter(result, year) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength
+          missed_years = []
+          result.each_slice(3) do |s| # ISO website only allows 3 connections
+            fetch_pages(s, 3).each do |r|
+              return { ret: r } if !year
+
+              r.date.select { |d| d.type == "published" }.each do |d|
+                return { ret: r } if year.to_i == d.at.to_date.year
+
+                missed_years << d.on(:year)
+              end
+            end
+          end
+          { years: missed_years }
+        end
+
+        # @param hits [RelatonBib::HitCollection<RelatonBib::Hit>]
+        # @param threads [Integer]
+        # @return [Array<RelatonBib::GbBibliographicItem>]
+        def fetch_pages(hits, threads)
+          workers = Core::WorkersPool.new threads
+          workers.worker { |w| { i: w[:i], hit: w[:hit].item } }
+          hits.each_with_index { |hit, i| workers << { i: i, hit: hit } }
+          workers.end
+          workers.result.sort_by { |x| x[:i] }.map { |x| x[:hit] }
+        end
+      end
+    end
+  end
+end

data/lib/relaton/gb/ccs.rb

@@ -0,0 +1,14 @@
+module Relaton
+  module Gb
+    class CCS < Bib::ICS
+      xml { root "ccs" }
+
+      def text
+        val = @text
+        return val if val && !val.empty?
+
+        Cnccs.fetch(code)&.description if code
+      end
+    end
+  end
+end

data/lib/relaton/gb/committee.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Gb
+    class Committee < Lutaml::Model::Serializable
+      attribute :type, :string, values: %w[technical provisional drafting]
+      attribute :content, :string
+
+      xml do
+        map_attribute "type", to: :type
+        map_content to: :content
+      end
+    end
+  end
+end

data/lib/relaton/gb/docidentifier.rb

@@ -0,0 +1,72 @@
+module Relaton
+  module Gb
+    # A Chinese Standard identifier backed by `Pubid::Gb`.
+    #
+    # `content` stays a plain string, so serialization is unchanged; the parsed
+    # identifier lives beside it in `@pubid` and drives the three mutators.
+    # Follows the IALA, CEN and OMG shape (`lib/relaton/iala/docidentifier.rb`,
+    # `lib/relaton/cen/model/docidentifier.rb`).
+    class Docidentifier < Bib::Docidentifier
+      # @return [Pubid::Gb::Identifier, nil] nil when the content is not a GB
+      #   identifier, or the grammar cannot read it
+      attr_reader :pubid
+
+      # Capture the inherited (LocalizedMarkedUpString) content setter before
+      # overriding #content=, so #replace_pubid writes the re-rendered string
+      # back WITHOUT re-parsing it and discarding the mutation.
+      alias_method :store_content, :content=
+
+      def content=(value)
+        super
+        return unless value
+
+        @pubid = begin
+          # `pubid` is required lazily because deserialization reaches this
+          # class without the flavor entry file having been loaded. LoadError
+          # degrades to a plain string; StandardError covers a value that is
+          # not a GB identifier. Both are DATA and must not raise. A malformed
+          # *query* raises — see Bibliography.get.
+          require "pubid"
+          ::Pubid::Gb::Identifier.parse(value)
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+
+      def remove_part!
+        return unless @pubid
+
+        replace_pubid @pubid.exclude(:part)
+      end
+
+      def remove_date!
+        return unless @pubid
+
+        replace_pubid @pubid.exclude(:year)
+      end
+
+      # Strips the part and the date, then wraps the identifier in pubid's
+      # `AllParts`. `Pubid::Gb::Identifiers::AllParts` renders the
+      # ` (all parts)` marker itself and parses the form back, so — unlike CEN
+      # and BSI, whose renderers have no marker — `content` is the wrapper's
+      # own string (`GB/T 5606 (all parts)`, which
+      # `spec/gb/fixtures/gbt_5606_2004_all_parts.xml` asserts).
+      #
+      # The guard keeps a second call from doubling the marker.
+      def to_all_parts!
+        return if !@pubid || @pubid.all_parts?
+
+        replace_pubid @pubid.exclude(:part, :year).to_all_parts
+      end
+
+      private
+
+      # `Pubid#exclude` returns a COPY, so the new identifier replaces the old
+      # one.
+      def replace_pubid(new_pubid)
+        @pubid = new_pubid
+        store_content @pubid.to_s
+      end
+    end
+  end
+end

data/lib/relaton/gb/doctype.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Gb
+    class Doctype < Bib::Doctype
+      TYPES = %w[standard recommendation].freeze
+
+      attribute :type, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/gb/ext.rb

@@ -0,0 +1,36 @@
+require_relative "doctype"
+require_relative "committee"
+require_relative "structured_identifier"
+require_relative "stage_name"
+require_relative "gb_type"
+require_relative "ccs"
+
+module Relaton
+  module Gb
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :subdoctype, :string, values: %w[specification method-of-test vocabulary code-of-practice]
+      attribute :structuredidentifier, StructuredIdentifier
+      attribute :stagename, StageName
+      attribute :gbtype, GbType
+      attribute :ccs, CCS, collection: true
+      attribute :plannumber, :string
+
+      xml do
+        map_element "stagename", to: :stagename
+        map_element "gbtype", to: :gbtype
+        map_element "ccs", to: :ccs
+        map_element "plannumber", to: :plannumber
+      end
+
+      key_value do
+        map_element "stagename", to: :stagename
+        map_element "gbtype", to: :gbtype
+        map_element "ccs", to: :ccs
+        map_element "plannumber", to: :plannumber
+      end
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-gb"]
+    end
+  end
+end

data/lib/relaton/gb/gb_scraper.rb

@@ -0,0 +1,61 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+require "nokogiri"
+require_relative "scraper"
+
+module Relaton
+  module Gb
+    # National standard scraper.
+    module GbScraper
+      extend Scraper
+      SEARCH_URL = "https://openstd.samr.gov.cn/bzgk/gb/std_list"
+      DOC_URL = "http://openstd.samr.gov.cn/bzgk/gb/newGbInfo?hcno="
+
+      class << self
+        # @param text [Strin] code of standard for serarch
+        # @return [RelatonGb::HitCollection]
+        def scrape_page(text) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+          doc = agent.get("#{SEARCH_URL}?p.p2=#{CGI.escape(text)}")
+          hits = doc.xpath(
+            "//table[contains(@class, 'result_list')]/tbody[2]/tr",
+          ).map do |h|
+            ref = h.at "./td[2]/a"
+            pid = ref[:onclick].match(/[0-9A-F]+/).to_s
+            status = h.at("./td[7]").text.strip
+            rdate = h.at("./td[8]").text.strip
+            Hit.new pid: pid, docref: ref.text, scraper: self,
+                    release_date: rdate, status: status
+          end
+          HitCollection.new hits.sort_by(&:release_date).reverse
+        rescue Mechanize::Error => e
+          raise Relaton::RequestError, e.message
+        end
+
+        def agent
+          @agent ||= Mechanize.new
+        end
+
+        # @param hit [RelatonGb::Hit] standard's page id
+        # @return [RelatonGb::GbBibliographicItem]
+        def scrape_doc(hit)
+          src = DOC_URL + hit.pid
+          doc = agent.get src
+          ItemData.new(**scrapped_data(doc, src, hit))
+        rescue Mechanize::Error => e
+          raise Relaton::RequestError, e.message
+        end
+
+        # @param doc [Nokogiri::HTML]
+        # @param _ref [String]
+        # @return [Hash]
+        #   * :type [String]
+        #   * :name [String]
+        # def get_committee(doc, _ref)
+        #   name = doc.at("//div[contains(., '归口单位') or contains(., '归口部门')]/following-sibling::div")
+        #   Committee.new(type: "technical", content: name.text.delete("\r\n\t\t"))
+        # end
+      end
+    end
+  end
+end

data/lib/relaton/gb/gb_type.rb

@@ -0,0 +1,20 @@
+module Relaton
+  module Gb
+    class GbType < Lutaml::Model::Serializable
+      attribute :scope, :string, values: %w[national sector professional local enterprise social-group]
+      attribute :prefix, :string
+      attribute :mandate, :string, values: %w[mandatory recommended guidelines]
+      attribute :topic, :string, values: %w[
+        basic health-and-safety environment-protection engineering-and-construction
+        product method management-techniques other
+      ]
+
+      xml do
+        map_element "gbscope", to: :scope
+        map_element "gbprefix", to: :prefix
+        map_element "gbmandate", to: :mandate
+        map_element "gbtopic", to: :topic
+      end
+    end
+  end
+end