relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/gb/hit.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Gb
+    # Hit.
+    class Hit < Relaton::Core::Hit
+      # @return [String]
+      attr_reader :pid, :docref
+
+      # @return [Date, NilClass]
+      attr_reader :release_date
+
+      # @return [String, NilClass]
+      attr_reader :status
+
+      # @return [RelatonGb::GbScraper, RelatonGb::SecScraper, RelatonGb::TScraper]
+      attr_reader :scraper
+
+      # @param pid [String]
+      # @param docref [String]
+      # @parma scraper [RelatonGb::GbScraper, RelatonGb::SecScraper, RelatonGb::TScraper]
+      # @param release_date [String]
+      # @status [String, NilClass]
+      # @param hit_collection [RelatonGb:HitCollection, NilClass]
+      def initialize(pid:, docref:, scraper:, **args)
+        @pid            = pid
+        @docref         = docref
+        @scraper        = scraper
+        @release_date   = Date.parse args[:release_date] if args[:release_date]
+        @status         = args[:status]
+        @hit_collection = args[:hit_collection]
+      end
+
+      # Parse page.
+      # @return [Isobib::IsoBibliographicItem]
+      def item
+        @item ||= scraper.scrape_doc self
+      end
+
+      # @return [String]
+      def inspect
+        "<#{self.class}:#{format('%<id>#.14x', id: object_id << 1)} " \
+          "@fullIdentifier=\"#{@fetch&.shortref}\" " \
+          "@docref=\"#{docref}\">"
+      end
+    end
+  end
+end

data/lib/relaton/gb/hit_collection.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "hit"
+
+module Relaton
+  module Gb
+    # Page of hit collection
+    class HitCollection < Relaton::Core::HitCollection
+      # @param hits [Array<Hash>]
+      # @param hit_pages [Integer]
+      # @param scraper [RelatonGb::GbScraper, RelatonGb::SecScraper,
+      #   RelatonGb::TScraper]
+      def initialize(hits = [])
+        @array = hits
+        @fetched = false
+      end
+    end
+  end
+end

data/lib/relaton/gb/item.rb

@@ -0,0 +1,13 @@
+require_relative "docidentifier"
+require_relative "ext"
+
+module Relaton
+  module Gb
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/gb/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Gb
+    class ItemData < Relaton::Bib::ItemData
+    end
+  end
+end

data/lib/relaton/gb/processor.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "relaton/core/processor"
+
+module Relaton
+  module Gb
+    class Processor < Relaton::Core::Processor
+      def initialize
+        @short = :relaton_gb
+        @prefix = "CN"
+        # Global prefixes come from Pubid::Gb.prefixes. "CN" is not one of
+        # them, so it leaves the register; routing still reads @prefix.
+        @pubid_flavor = :Gb
+        @defaultprefix = %r{^(GB|GB/T|GB/Z) }
+        @idtype = "Chinese Standard"
+      end
+
+      # @param code [String]
+      # @param date [String, NilClass] year
+      # @param opts [Hash]
+      # @return [Relaton::Gb::ItemData, nil]
+      def get(code, date, opts)
+        require_relative "../gb"
+        Bibliography.get(code, date, opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Gb::ItemData]
+      def from_xml(xml)
+        require_relative "../gb"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Gb::ItemData]
+      def from_yaml(yaml)
+        require_relative "../gb" # defines Item — cache reads hit this cold
+        Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../gb"
+        @grammar_hash ||= ::Relaton::Gb.grammar_hash
+      end
+    end
+  end
+end

data/lib/relaton/gb/project_number.rb

@@ -0,0 +1,38 @@
+module Relaton
+  module Gb
+    class ProjectNumber < Lutaml::Model::Serializable
+      attribute :part, :integer
+      attribute :subpart, :integer
+      attribute :amendment, :integer
+      attribute :corrigendum, :integer
+      attribute :origyr, :string
+      attribute :content, :string
+
+      xml do
+        root "project-ntnumber"
+        map_attribute "part", to: :part
+        map_attribute "subpart", to: :subpart
+        map_attribute "amendment", to: :amendment
+        map_attribute "corrigendum", to: :corrigendum
+        map_attribute "origyr", to: :origyr
+        map_content to: :content
+      end
+
+      def to_all_parts!
+        remove_part!
+        remove_date!
+      end
+
+      def remove_part!
+        self.part = nil
+        self.subpart = nil
+        self.amendment = nil
+        self.corrigendum = nil
+      end
+
+      def remove_date!
+        self.origyr = nil
+      end
+    end
+  end
+end

data/lib/relaton/gb/scraper.rb

@@ -0,0 +1,221 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+require "yaml"
+require "gb_agencies"
+
+module Relaton
+  module Gb
+    # Common scrapping methods.
+    module Scraper
+      STAGES = { "即将实施" => "published",
+                "现行" => "activated",
+                "废止" => "obsoleted",
+                "被代替" => "replaced" }.freeze
+
+      @prefixes = nil
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @param src [String]
+      # @param hit [RelatonGb::Hit]
+      # @return [Hash]
+      def scrapped_data(doc, src, hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        {
+          fetched: Date.today.to_s,
+          type: "standard",
+          docidentifier: get_docid(hit.docref),
+          title: get_titles(doc),
+          contributor: get_contributors(doc, hit.docref),
+          status: get_status(doc, hit.status),
+          source: get_source(src),
+          date: get_dates(doc),
+          language: ["zh"],
+          script: ["Hans"],
+          ext: get_ext(doc, hit.docref),
+        }
+      end
+
+      # @param docref [String]
+      # @return [Array<Relaton::Bib::Docidentifier>]
+      def get_docid(docref)
+        [Docidentifier.new(content: docref, type: "Chinese Standard", primary: true)]
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @param docref [Strings]
+      # @return [Array<Relaton::Bib::Contributor>]
+      def get_contributors(doc, docref)
+        name = docref_pubid(docref)&.publisher&.to_s
+        gbtype = get_gbtype(doc, docref)
+        org_names = %w[en zh].map { |l| create_org_name(l, name, gbtype) }.compact
+        return [] unless org_names.any?
+
+        org = Bib::Organization.new name: org_names
+        role = Bib::Contributor::Role.new type: "publisher"
+        [Bib::Contributor.new(organization: org, role: [role])]
+      end
+
+      # @param lang [String]
+      # @param name [String]
+      # @param gbtype [Hash]
+      # @return [Relaton::Bib::TypedLocalizedString, nil]
+      def create_org_name(lang, name, gbtype)
+        ag = GbAgencies::Agencies.new(lang, {}, "")
+        content = ag.standard_agency1(gbtype.scope, name, gbtype.mandate)
+        return unless content
+
+        Bib::TypedLocalizedString.new language: lang, content: content
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Relaton::Bib::Title>]
+      def get_titles(doc)
+        tzh = doc.at("//td[contains(text(), '中文标准名称')]/b").text
+        titles = Relaton::Bib::Title.from_string tzh, "zh", "Hans"
+        ten = doc.at("//td[contains(text(), '英文标准名称')]").text.match(/[\w\s]+/).to_s
+        return titles if ten.empty?
+
+        titles + Relaton::Bib::Title.from_string(ten, "en", "Latn")
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @param status [String, NilClass]
+      # @return [Relaton::Bib::Status]
+      def get_status(doc, status = nil)
+        status ||= doc.at("//td[contains(., '标准状态')]/span")&.text&.strip
+        return unless STAGES[status]
+
+        stage = Bib::Status::Stage.new content: STAGES[status]
+        Bib::Status.new stage: stage
+      end
+
+      private
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<String>]
+      def get_ccs(doc)
+        code = doc.at("//div[contains(text(), '中国标准分类号')]/following-sibling::div").text.strip
+        [CCS.new(code: code)]
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Relaton::Bib::ICS>]
+      def get_ics(doc)
+        ics = doc.at("//div[contains(text(), '国际标准分类号')]/following-sibling::div"\
+                    " | //dt[contains(text(), '国际标准分类号')]/following-sibling::dd")
+        return [] unless ics
+
+        code = ics.text.strip
+        [Bib::ICS.new(code: code)]
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [String]
+      def get_scope(doc)
+        issued = doc.at("//div[contains(., '发布单位')]/following-sibling::div")
+        case issued&.text
+        when /国家标准/ then "national"
+        when /^行业标准/ then "sector"
+        end
+      end
+
+      # The publisher of a social group standard names the group
+      # (`T/GZAEPI`), so only its first segment keys `yaml/prefixes.yaml`.
+      #
+      # @param ref [String]
+      # @return [Hash{String=>String}, nil]
+      def get_prefix(ref)
+        pref = docref_pubid(ref)&.publisher&.to_s
+        prefix pref.split("/").first if pref
+      end
+
+      # @param pref [String]
+      # @return [Hash{String=>String}]
+      def prefix(pref)
+        @prefixes ||= YAML.load_file File.join(__dir__, "yaml/prefixes.yaml")
+        @prefixes[pref]
+      end
+
+      # @param ref [String]
+      # @return [String]
+      def get_mandate(ref)
+        case docref_pubid(ref)&.mandate
+        when "T" then "recommended"
+        when "Z" then "guidelines"
+        else "mandatory"
+        end
+      end
+
+      def get_source(src)
+        (Bib::Uri.new(type: "src", content: src))
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      #   * :type [String] type of date
+      #   * :on [String] date
+      def get_dates(doc)
+        date = doc.at("//div[contains(text(), '发布日期')]/following-sibling::div"\
+                      " | //dt[contains(text(), '发布日期')]/following-sibling::dd")
+        [Bib::Date.new(type: "published", at: date.text.delete("\r\n\t\t"))]
+      end
+
+      def get_ext(doc, docref)
+        Ext.new(
+          doctype: get_type,
+          gbtype: get_gbtype(doc, docref),
+          flavor: "gb",
+          # gbcommittee: get_committee(doc, docref),
+          ccs: get_ccs(doc),
+          ics: get_ics(doc),
+          structuredidentifier: fetch_structuredidentifier(docref),
+          plannumber: parse_docref(docref)[0],
+        )
+      end
+
+      def get_type
+        Doctype.new content: "standard"
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @param ref [String]
+      # @return [Relaton::Gb::GbType]
+      def get_gbtype(doc, ref)
+        # ref = get_ref(doc)
+        GbType.new(scope: get_scope(doc), prefix: get_prefix(ref)&.fetch("prefix", nil), mandate: get_mandate(ref), topic: "other")
+      end
+
+      # A portal docref is data, so a value pubid cannot read is nil rather
+      # than an error.
+      #
+      # @param docref [String]
+      # @return [Pubid::Gb::Identifier, nil]
+      def docref_pubid(docref)
+        require "pubid"
+        ::Pubid::Gb::Identifier.parse(docref)
+      rescue StandardError
+        nil
+      end
+
+      # @param docref [String]
+      # @return [Array<String, nil>] the id without part and year (e.g.
+      #   `GB/T 5606`), the part, and the year
+      def parse_docref(docref)
+        pubid = docref_pubid(docref)
+        return [docref, nil, nil] unless pubid
+
+        [pubid.exclude(:part, :year).to_s, pubid.part, pubid.year]
+      end
+
+      # @param docref [String]
+      # @return [Relaton::Iso::StructuredIdentifier]
+      def fetch_structuredidentifier(docref)
+        docnumber, partnumber, origyr = parse_docref(docref)
+        project_number = ProjectNumber.new(part: partnumber, origyr: origyr, content: docref)
+        StructuredIdentifier.new(
+          type: "Chinese Standard", tc_document_number: docnumber, project_number: project_number,
+        )
+      end
+    end
+  end
+end

data/lib/relaton/gb/sec_scraper.rb

@@ -0,0 +1,92 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "nokogiri"
+require_relative "scraper"
+require_relative "item"
+require_relative "hit_collection"
+require_relative "hit"
+
+module Relaton
+  module Gb
+    # Sector standard scraper
+    module SecScraper
+      extend Scraper
+      extend Core::ArrayWrapper
+
+      class << self
+        # @param text [String] code of standard for serarch
+        # @return [Relaton::Gb::HitCollection]
+        def scrape_page(text)
+          # uri = URI "http://www.std.gov.cn/hb/search/hbPage?searchText=#{text}"
+          uri = URI "https://hbba.sacinfo.org.cn/stdQueryList"
+          resp = Net::HTTP.post uri, URI.encode_www_form({ key: text }),
+                                "Content-Type" => "application/x-www-form-urlencoded"
+          # res = JSON.parse Net::HTTP.get(uri)
+          json = JSON.parse resp.body
+          hits = json["records"].map do |h|
+            Hit.new pid: h["pk"], docref: h["code"], status: h["status"], scraper: self
+          end
+          # hits = res["rows"].map do |r|
+          #   Hit.new pid: r["id"], title: r["STD_CODE"], scraper: self
+          # end
+          HitCollection.new hits
+        rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+              Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
+              OpenSSL::SSL::SSLError, Errno::ETIMEDOUT, Net::OpenTimeout
+          raise Relaton::RequestError, "Cannot access #{uri}"
+        end
+
+        # @param hit [Relaton::Gb::Hit]
+        # @return [Relaton::Gb::ItemData]
+        def scrape_doc(hit)
+          src = "https://hbba.sacinfo.org.cn/stdDetail/#{hit.pid}"
+          page_uri = URI src
+          doc = Nokogiri::HTML Net::HTTP.get(page_uri)
+          ItemData.new(**scrapped_data(doc, src, hit))
+        rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+              Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
+              OpenSSL::SSL::SSLError, Errno::ETIMEDOUT, Net::OpenTimeout
+          raise Relaton::RequestError, "Cannot access #{src}"
+        end
+
+        private
+
+        # @param doc [Nokogiri::HTML::Document]
+        # @return [Array<Relaton::Bib::Title>]
+        def get_titles(doc)
+          tzh = doc.at("//h4").text.delete("\r\n\t")
+          Bib::Title.from_string(tzh, "zh", "Hans")
+        end
+
+        # @param _doc [Nokogiri::HTML::Document]
+        # @param ref [String]
+        # @return [Hash]
+        #   * :type [String]
+        #   * :name [String]
+        # def get_committee(_doc, ref)
+        #   # ref = get_ref(doc)
+        #   name = get_prefix(ref)["administration"]
+        #   { type: "technical", name: name }
+        # end
+
+        # @param _doc [Nokogiri::HTML::Document]
+        # @return [String]
+        def get_scope(_doc)
+          "sector"
+        end
+
+        # @param doc [Nokogiri::HTML::Document]
+        # @return [Array<String>]
+        def get_ccs(doc)
+          array(doc.at("//dt[contains(text(), '中国标准分类号')]/following-sibling::dd")).map do |cc|
+            text = Cnccs.fetch(cc.text.strip)&.description
+            CCS.new code: cc.text, text: text
+          end
+        end
+      end
+    end
+  end
+end

data/lib/relaton/gb/stage_name.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Gb
+    class StageName < Lutaml::Model::Serializable
+      attribute :abbreviation, :string
+      attribute :content, :string
+
+      xml do
+        map_attribute "abbreviation", to: :abbreviation
+        map_content to: :content
+      end
+    end
+  end
+end

data/lib/relaton/gb/structured_identifier.rb

@@ -0,0 +1,26 @@
+require_relative "project_number"
+
+module Relaton
+  module Gb
+    class StructuredIdentifier < Lutaml::Model::Serializable
+      attribute :type, :string
+      attribute :project_number, ProjectNumber
+      attribute :tc_document_number, :integer
+
+      xml do
+        root "structuredidentifier"
+        map_attribute "type", to: :type
+        map_element "project-number", to: :project_number
+        map_element "tc-document-number", to: :tc_document_number
+      end
+
+      def to_all_parts!
+        project_number&.to_all_parts!
+      end
+
+      def remove_date!
+        project_number&.remove_date!
+      end
+    end
+  end
+end

data/lib/relaton/gb/t_scraper.rb

@@ -0,0 +1,126 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+require "nokogiri"
+require_relative "scraper"
+require_relative "hit_collection"
+require_relative "hit"
+
+module Relaton
+  module Gb
+    # Social standard scarpper.
+    module TScraper
+      extend Scraper
+
+      class << self
+        # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        # @param text [String]
+        # @return [Relaton::Gb::HitCollection]
+        def scrape_page(text)
+          url = "http://www.ttbz.org.cn/Home/Standard?searchType=2&key=" \
+                "#{CGI.escape(text.tr('-', [8212].pack('U')))}"
+          doc = agent.get(url)
+          xpath = '//table[contains(@class, "standard_list_table")]/tr/td/a'
+          t_xpath = "../preceding-sibling::td[4]"
+          hits = doc.xpath(xpath).map do |h|
+            docref = h.at(t_xpath).text.gsub(/â\u0080\u0094/, "-")
+            status = h.at("../preceding-sibling::td[1]").text.delete "\r\n"
+            pid = h[:href].sub(%r{/$}, "")
+            Hit.new pid: pid, docref: docref, status: status, scraper: self
+          end
+          HitCollection.new hits
+        rescue Mechanize::ResponseCodeError => e
+          return nil if e.response_code == "404"
+
+          raise Relaton::RequestError, "Cannot access #{url}: #{e.message}"
+        rescue Mechanize::Error => e
+          raise Relaton::RequestError, "Cannot access #{url}: #{e.message}"
+        end
+        # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+        # @param hit [RelatonGb::Hit] standard's page path
+        # @return [RelatonGb::GbBibliographicItem]
+        def scrape_doc(hit)
+          src = "http://www.ttbz.org.cn#{hit.pid}"
+          doc = agent.get(src)
+          ItemData.new(**scrapped_data(doc, src, hit))
+        rescue Mechanize::Error => e
+          raise Relaton::RequestError, "Cannot access #{src}: #{e.message}"
+        end
+
+        def agent
+          @agent ||= Mechanize.new
+        end
+
+        private
+
+        # rubocop:disable Metrics/MethodLength
+        # @param doc [Nokogiri::HTML::Document]
+        # @param src [String]
+        # @param hit [RelatonGb::Hit]
+        # @return [Hash]
+        def scrapped_data(doc, src, hit)
+          # docid_xpt  = '//td[contains(.,"标准编号")]/following-sibling::td[1]'
+          # status_xpt = '//td[contains(.,"标准状态")]/following-sibling::td[1]/span'
+          {
+            # committee: get_committee(doc, hit.docref),
+            docid: get_docid(hit.docref),
+            title: get_titles(doc),
+            doctype: get_type,
+            docstatus: get_status(doc, hit.status),
+            gbtype: gbtype,
+            ccs: get_ccs(doc),
+            ics: get_ics(doc),
+            link: [{ type: "src", content: src }],
+            date: get_dates(doc),
+            language: ["zh"],
+            script: ["Hans"],
+            structuredidentifier: fetch_structuredidentifier(hit.docref),
+          }
+        end
+        # rubocop:enable Metrics/MethodLength
+
+        # def get_committee(doc, _ref)
+        #   {
+        #     name: doc.xpath('//td[.="团体名称"]/following-sibling::td[1]').text,
+        #     type: "technical",
+        #   }
+        # end
+
+        def get_titles(doc)
+          xpz = '//td[contains(.,"中文标题")]/following-sibling::td[1]'
+          titles = Bib::Title.from_string doc.at(xpz)
+            .text, "zh", "Hans"
+          xpe = '//td[contains(.,"英文标题")]/following-sibling::td[1]'
+          ten = doc.xpath(xpe).text
+          return titles if ten.empty?
+
+          titles + Bib::Title.from_string(ten, "en", "Latn")
+        end
+
+        def gbtype
+          { scope: "social-group", prefix: "T", mandate: "mandatory",
+            topic: "other" }
+        end
+
+        def get_ccs(doc)
+          [doc.xpath('//td[contains(.,"中国标准分类号")]/following-sibling::td[1]')
+            .text.gsub(/[\r\n]/, "").strip.match(/^[^\s]+/).to_s]
+        end
+
+        def get_ics(doc)
+          xpath = '//td[contains(.,"国际标准分类号")]/following-sibling::td[1]/span'
+          ics = doc.xpath(xpath).text.match(/^[^\s]+/).to_s
+          field, group, subgroup = ics.split "."
+          [{ field: field, group: group.ljust(3, "0"), subgroup: subgroup }]
+        end
+
+        def get_dates(doc)
+          d = doc.xpath('//td[contains(.,"发布日期")]/following-sibling::td[1]/span')
+            .text.match(/(?<y>\d{4})[^\d]+(?<m>\d{2})[^\d]+(?<d>\d{2})/)
+          [{ type: "published", on: "#{d[:y]}-#{d[:m]}-#{d[:d]}" }]
+        end
+      end
+    end
+  end
+end

data/lib/relaton/gb/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Gb
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-gb".freeze
+    end
+  end
+end