relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/gb/scraper.rb

@@ -0,0 +1,204 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+require "yaml"
+require "gb_agencies"
+
+module Relaton
+  module Gb
+    # Common scrapping methods.
+    module Scraper
+      STAGES = { "即将实施" => "published",
+                "现行" => "activated",
+                "废止" => "obsoleted",
+                "被代替" => "replaced" }.freeze
+
+      @prefixes = nil
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @param src [String]
+      # @param hit [RelatonGb::Hit]
+      # @return [Hash]
+      def scrapped_data(doc, src, hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        {
+          fetched: Date.today.to_s,
+          type: "standard",
+          docidentifier: get_docid(hit.docref),
+          title: get_titles(doc),
+          contributor: get_contributors(doc, hit.docref),
+          status: get_status(doc, hit.status),
+          source: get_source(src),
+          date: get_dates(doc),
+          language: ["zh"],
+          script: ["Hans"],
+          ext: get_ext(doc, hit.docref),
+        }
+      end
+
+      # @param docref [String]
+      # @return [Array<Relaton::Bib::Docidentifier>]
+      def get_docid(docref)
+        [Docidentifier.new(content: docref, type: "Chinese Standard", primary: true)]
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @param docref [Strings]
+      # @return [Array<Relaton::Bib::Contributor>]
+      def get_contributors(doc, docref)
+        name = docref.match(/^[^\s]+/).to_s
+        name.sub!(%r{/[TZ]$}, "") unless name =~ /^GB/
+        gbtype = get_gbtype(doc, docref)
+        org_names = %w[en zh].map { |l| create_org_name(l, name, gbtype) }.compact
+        return [] unless org_names.any?
+
+        org = Bib::Organization.new name: org_names
+        role = Bib::Contributor::Role.new type: "publisher"
+        [Bib::Contributor.new(organization: org, role: [role])]
+      end
+
+      # @param lang [String]
+      # @param name [String]
+      # @param gbtype [Hash]
+      # @return [Relaton::Bib::TypedLocalizedString, nil]
+      def create_org_name(lang, name, gbtype)
+        ag = GbAgencies::Agencies.new(lang, {}, "")
+        content = ag.standard_agency1(gbtype.scope, name, gbtype.mandate)
+        return unless content
+
+        Bib::TypedLocalizedString.new language: lang, content: content
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Relaton::Bib::Title>]
+      def get_titles(doc)
+        tzh = doc.at("//td[contains(text(), '中文标准名称')]/b").text
+        titles = Relaton::Bib::Title.from_string tzh, "zh", "Hans"
+        ten = doc.at("//td[contains(text(), '英文标准名称')]").text.match(/[\w\s]+/).to_s
+        return titles if ten.empty?
+
+        titles + Relaton::Bib::Title.from_string(ten, "en", "Latn")
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @param status [String, NilClass]
+      # @return [Relaton::Bib::Status]
+      def get_status(doc, status = nil)
+        status ||= doc.at("//td[contains(., '标准状态')]/span")&.text&.strip
+        return unless STAGES[status]
+
+        stage = Bib::Status::Stage.new content: STAGES[status]
+        Bib::Status.new stage: stage
+      end
+
+      private
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<String>]
+      def get_ccs(doc)
+        code = doc.at("//div[contains(text(), '中国标准分类号')]/following-sibling::div").text.strip
+        [CCS.new(code: code)]
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Relaton::Bib::ICS>]
+      def get_ics(doc)
+        ics = doc.at("//div[contains(text(), '国际标准分类号')]/following-sibling::div"\
+                    " | //dt[contains(text(), '国际标准分类号')]/following-sibling::dd")
+        return [] unless ics
+
+        code = ics.text.strip
+        [Bib::ICS.new(code: code)]
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [String]
+      def get_scope(doc)
+        issued = doc.at("//div[contains(., '发布单位')]/following-sibling::div")
+        case issued&.text
+        when /国家标准/ then "national"
+        when /^行业标准/ then "sector"
+        end
+      end
+
+      # @param ref [String]
+      # @return [String]
+      def get_prefix(ref)
+        pref = ref.match(/^[^\s]+/).to_s.split("/").first
+        prefix pref
+      end
+
+      # @param pref [String]
+      # @return [Hash{String=>String}]
+      def prefix(pref)
+        @prefixes ||= YAML.load_file File.join(__dir__, "yaml/prefixes.yaml")
+        @prefixes[pref]
+      end
+
+      # @param ref [String]
+      # @return [String]
+      def get_mandate(ref)
+        case ref.match(%r{(?<=\/)[^\s]+}).to_s
+        when "T" then "recommended"
+        when "Z" then "guidelines"
+        else "mandatory"
+        end
+      end
+
+      def get_source(src)
+        (Bib::Uri.new(type: "src", content: src))
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      #   * :type [String] type of date
+      #   * :on [String] date
+      def get_dates(doc)
+        date = doc.at("//div[contains(text(), '发布日期')]/following-sibling::div"\
+                      " | //dt[contains(text(), '发布日期')]/following-sibling::dd")
+        [Bib::Date.new(type: "published", at: date.text.delete("\r\n\t\t"))]
+      end
+
+      def get_ext(doc, docref)
+        Ext.new(
+          doctype: get_type,
+          gbtype: get_gbtype(doc, docref),
+          flavor: "gb",
+          # gbcommittee: get_committee(doc, docref),
+          ccs: get_ccs(doc),
+          ics: get_ics(doc),
+          structuredidentifier: fetch_structuredidentifier(docref),
+          plannumber: parse_docref(docref)[0],
+        )
+      end
+
+      def get_type
+        Doctype.new content: "standard"
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @param ref [String]
+      # @return [Relaton::Gb::GbType]
+      def get_gbtype(doc, ref)
+        # ref = get_ref(doc)
+        GbType.new(scope: get_scope(doc), prefix: get_prefix(ref)["prefix"], mandate: get_mandate(ref), topic: "other")
+      end
+
+      # @param docref [String]
+      # @return [Array<String>] array of matched groups [docnumber, partnumber]
+      def parse_docref(docref)
+        m = docref.match(/^([^–—.-]*\d+)\.?((?<=\.)\d+|)(?:-(\d{4}))?/)
+        [m[1], m[2], m[3]]
+      end
+
+      # @param docref [String]
+      # @return [Relaton::Iso::StructuredIdentifier]
+      def fetch_structuredidentifier(docref)
+        docnumber, partnumber, origyr = parse_docref(docref)
+        project_number = ProjectNumber.new(part: partnumber, origyr: origyr, content: docref)
+        StructuredIdentifier.new(
+          type: "Chinese Standard", tc_document_number: docnumber, project_number: project_number,
+        )
+      end
+    end
+  end
+end

data/lib/relaton/gb/sec_scraper.rb

@@ -0,0 +1,92 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "nokogiri"
+require_relative "scraper"
+require_relative "item"
+require_relative "hit_collection"
+require_relative "hit"
+
+module Relaton
+  module Gb
+    # Sector standard scraper
+    module SecScraper
+      extend Scraper
+      extend Core::ArrayWrapper
+
+      class << self
+        # @param text [String] code of standard for serarch
+        # @return [Relaton::Gb::HitCollection]
+        def scrape_page(text)
+          # uri = URI "http://www.std.gov.cn/hb/search/hbPage?searchText=#{text}"
+          uri = URI "https://hbba.sacinfo.org.cn/stdQueryList"
+          resp = Net::HTTP.post uri, URI.encode_www_form({ key: text }),
+                                "Content-Type" => "application/x-www-form-urlencoded"
+          # res = JSON.parse Net::HTTP.get(uri)
+          json = JSON.parse resp.body
+          hits = json["records"].map do |h|
+            Hit.new pid: h["pk"], docref: h["code"], status: h["status"], scraper: self
+          end
+          # hits = res["rows"].map do |r|
+          #   Hit.new pid: r["id"], title: r["STD_CODE"], scraper: self
+          # end
+          HitCollection.new hits
+        rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+              Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
+              OpenSSL::SSL::SSLError, Errno::ETIMEDOUT, Net::OpenTimeout
+          raise Relaton::RequestError, "Cannot access #{uri}"
+        end
+
+        # @param hit [Relaton::Gb::Hit]
+        # @return [Relaton::Gb::ItemData]
+        def scrape_doc(hit)
+          src = "https://hbba.sacinfo.org.cn/stdDetail/#{hit.pid}"
+          page_uri = URI src
+          doc = Nokogiri::HTML Net::HTTP.get(page_uri)
+          ItemData.new(**scrapped_data(doc, src, hit))
+        rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+              Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
+              OpenSSL::SSL::SSLError, Errno::ETIMEDOUT, Net::OpenTimeout
+          raise Relaton::RequestError, "Cannot access #{src}"
+        end
+
+        private
+
+        # @param doc [Nokogiri::HTML::Document]
+        # @return [Array<Relaton::Bib::Title>]
+        def get_titles(doc)
+          tzh = doc.at("//h4").text.delete("\r\n\t")
+          Bib::Title.from_string(tzh, "zh", "Hans")
+        end
+
+        # @param _doc [Nokogiri::HTML::Document]
+        # @param ref [String]
+        # @return [Hash]
+        #   * :type [String]
+        #   * :name [String]
+        # def get_committee(_doc, ref)
+        #   # ref = get_ref(doc)
+        #   name = get_prefix(ref)["administration"]
+        #   { type: "technical", name: name }
+        # end
+
+        # @param _doc [Nokogiri::HTML::Document]
+        # @return [String]
+        def get_scope(_doc)
+          "sector"
+        end
+
+        # @param doc [Nokogiri::HTML::Document]
+        # @return [Array<String>]
+        def get_ccs(doc)
+          array(doc.at("//dt[contains(text(), '中国标准分类号')]/following-sibling::dd")).map do |cc|
+            text = Cnccs.fetch(cc.text.strip)&.description
+            CCS.new code: cc.text, text: text
+          end
+        end
+      end
+    end
+  end
+end

data/lib/relaton/gb/stage_name.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Gb
+    class StageName < Lutaml::Model::Serializable
+      attribute :abbreviation, :string
+      attribute :content, :string
+
+      xml do
+        map_attribute "abbreviation", to: :abbreviation
+        map_content to: :content
+      end
+    end
+  end
+end

data/lib/relaton/gb/structured_identifier.rb

@@ -0,0 +1,26 @@
+require_relative "project_number"
+
+module Relaton
+  module Gb
+    class StructuredIdentifier < Lutaml::Model::Serializable
+      attribute :type, :string
+      attribute :project_number, ProjectNumber
+      attribute :tc_document_number, :integer
+
+      xml do
+        root "structuredidentifier"
+        map_attribute "type", to: :type
+        map_element "project-number", to: :project_number
+        map_element "tc-document-number", to: :tc_document_number
+      end
+
+      def to_all_parts!
+        project_number&.to_all_parts!
+      end
+
+      def remove_date!
+        project_number&.remove_date!
+      end
+    end
+  end
+end

data/lib/relaton/gb/t_scraper.rb

@@ -0,0 +1,126 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+require "nokogiri"
+require_relative "scraper"
+require_relative "hit_collection"
+require_relative "hit"
+
+module Relaton
+  module Gb
+    # Social standard scarpper.
+    module TScraper
+      extend Scraper
+
+      class << self
+        # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        # @param text [String]
+        # @return [Relaton::Gb::HitCollection]
+        def scrape_page(text)
+          url = "http://www.ttbz.org.cn/Home/Standard?searchType=2&key=" \
+                "#{CGI.escape(text.tr('-', [8212].pack('U')))}"
+          doc = agent.get(url)
+          xpath = '//table[contains(@class, "standard_list_table")]/tr/td/a'
+          t_xpath = "../preceding-sibling::td[4]"
+          hits = doc.xpath(xpath).map do |h|
+            docref = h.at(t_xpath).text.gsub(/â\u0080\u0094/, "-")
+            status = h.at("../preceding-sibling::td[1]").text.delete "\r\n"
+            pid = h[:href].sub(%r{/$}, "")
+            Hit.new pid: pid, docref: docref, status: status, scraper: self
+          end
+          HitCollection.new hits
+        rescue Mechanize::ResponseCodeError => e
+          return nil if e.response_code == "404"
+
+          raise Relaton::RequestError, "Cannot access #{url}: #{e.message}"
+        rescue Mechanize::Error => e
+          raise Relaton::RequestError, "Cannot access #{url}: #{e.message}"
+        end
+        # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+        # @param hit [RelatonGb::Hit] standard's page path
+        # @return [RelatonGb::GbBibliographicItem]
+        def scrape_doc(hit)
+          src = "http://www.ttbz.org.cn#{hit.pid}"
+          doc = agent.get(src)
+          ItemData.new(**scrapped_data(doc, src, hit))
+        rescue Mechanize::Error => e
+          raise Relaton::RequestError, "Cannot access #{src}: #{e.message}"
+        end
+
+        def agent
+          @agent ||= Mechanize.new
+        end
+
+        private
+
+        # rubocop:disable Metrics/MethodLength
+        # @param doc [Nokogiri::HTML::Document]
+        # @param src [String]
+        # @param hit [RelatonGb::Hit]
+        # @return [Hash]
+        def scrapped_data(doc, src, hit)
+          # docid_xpt  = '//td[contains(.,"标准编号")]/following-sibling::td[1]'
+          # status_xpt = '//td[contains(.,"标准状态")]/following-sibling::td[1]/span'
+          {
+            # committee: get_committee(doc, hit.docref),
+            docid: get_docid(hit.docref),
+            title: get_titles(doc),
+            doctype: get_type,
+            docstatus: get_status(doc, hit.status),
+            gbtype: gbtype,
+            ccs: get_ccs(doc),
+            ics: get_ics(doc),
+            link: [{ type: "src", content: src }],
+            date: get_dates(doc),
+            language: ["zh"],
+            script: ["Hans"],
+            structuredidentifier: fetch_structuredidentifier(hit.docref),
+          }
+        end
+        # rubocop:enable Metrics/MethodLength
+
+        # def get_committee(doc, _ref)
+        #   {
+        #     name: doc.xpath('//td[.="团体名称"]/following-sibling::td[1]').text,
+        #     type: "technical",
+        #   }
+        # end
+
+        def get_titles(doc)
+          xpz = '//td[contains(.,"中文标题")]/following-sibling::td[1]'
+          titles = Bib::Title.from_string doc.at(xpz)
+            .text, "zh", "Hans"
+          xpe = '//td[contains(.,"英文标题")]/following-sibling::td[1]'
+          ten = doc.xpath(xpe).text
+          return titles if ten.empty?
+
+          titles + Bib::Title.from_string(ten, "en", "Latn")
+        end
+
+        def gbtype
+          { scope: "social-group", prefix: "T", mandate: "mandatory",
+            topic: "other" }
+        end
+
+        def get_ccs(doc)
+          [doc.xpath('//td[contains(.,"中国标准分类号")]/following-sibling::td[1]')
+            .text.gsub(/[\r\n]/, "").strip.match(/^[^\s]+/).to_s]
+        end
+
+        def get_ics(doc)
+          xpath = '//td[contains(.,"国际标准分类号")]/following-sibling::td[1]/span'
+          ics = doc.xpath(xpath).text.match(/^[^\s]+/).to_s
+          field, group, subgroup = ics.split "."
+          [{ field: field, group: group.ljust(3, "0"), subgroup: subgroup }]
+        end
+
+        def get_dates(doc)
+          d = doc.xpath('//td[contains(.,"发布日期")]/following-sibling::td[1]/span')
+            .text.match(/(?<y>\d{4})[^\d]+(?<m>\d{2})[^\d]+(?<d>\d{2})/)
+          [{ type: "published", on: "#{d[:y]}-#{d[:m]}-#{d[:d]}" }]
+        end
+      end
+    end
+  end
+end

data/lib/relaton/gb/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Gb
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-gb".freeze
+    end
+  end
+end

data/lib/relaton/gb/yaml/prefixes.yaml

@@ -0,0 +1,200 @@
+# GbStandardNationalPrefix
+GB:
+  prefix: GB_national
+GBn:
+  prefix: GBn_confidential
+GJB:
+  prefix: GJB_military
+GSB:
+  prefix: GSB_physical
+
+# GbStandardSectorPrefix
+NY:
+  prefix: NY_agriculture
+  administration: 农业部
+SC:
+  prefix: SC_aquatic
+  administration: 农业部
+SL:
+  prefix: SL_water_resources
+  administration: 水利部
+QB:
+  prefix: QB_light_industry
+  administration: 国家发改委
+FZ:
+  prefix: FZ_textile
+  administration: 国家发改委
+YY:
+  prefix: YY_medicine
+  administration: 国家食品药品监督管理局
+MZ:
+  prefix: MZ_civil_affairs
+  administration: 民政部
+JY:
+  prefix: JY_education
+  administration: 教育部
+YC:
+  prefix: YC_tobacco
+  administration: 国家烟草专卖局
+YB:
+  prefix: YB_ferrous_metallurgy
+  administration: 国家发改委
+YS:
+  prefix: YS_nonferrous_metallurgy
+  administration: 国家发改委
+SY:
+  prefix: SY_natural_gas
+  administration: 国家发改委
+HG:
+  prefix: HG_chemical_industry
+  administration: 国家发改委
+SH:
+  prefix: SH_petrochemical
+  administration: 国家发改委
+JC:
+  prefix: JC_building_materials
+  administration: 国家发改委
+DZ:
+  prefix: DZ_geological
+  administration: 国土资源部
+TD:
+  prefix: TD_land_management
+  administration: 国土资源部
+CH:
+  prefix: CH_surveying
+  administration: 国家测绘局
+JB:
+  prefix: JB_mechanical
+  administration: 国家发改委
+QC:
+  prefix: QC_automotile
+  administration: 国家发改委
+MH:
+  prefix: MH_civil_aviation
+  administration: 中国民航管理总局
+WJ:
+  prefix: WJ_ordnance
+  administration: 国防科学工业委员会
+CB:
+  prefix: CB_ships
+  administration: 国防科学工业委员会
+HB:
+  prefix: HB_aviation
+  administration: 国防科学工业委员会
+QJ:
+  prefix: QJ_aerospace
+  administration: 国防科学工业委员会
+EJ:
+  prefix: EJ_nuclear_industry
+  administration: 国防科学工业委员会
+TB:
+  prefix: TB_rail_transport
+  administration: 铁道部
+JT:
+  prefix: JT_traffic
+  administration: 交通部
+LD:
+  prefix: LD_work_safety
+  administration: 劳动和社会保障部
+SJ:
+  prefix: SJ_electronics
+  administration: 信息产业部
+YD:
+  prefix: YD_communication
+  administration: 信息产业部
+GY:
+  prefix: GY_media
+  administration: 国家广播电视总局
+DY:
+  prefix: DY_film
+  administration: 国家电影局
+DL:
+  prefix: DL_electricity
+  administration: 国家发改委
+JR:
+  prefix: JR_financial
+  administration: 中国人民银行
+HY:
+  prefix: HY_oceanic
+  administration: 国家海洋局
+DA:
+  prefix: DA_archiving
+  administration: 国家档案局
+SN:
+  prefix: SN_inspection
+  administration: 国家质量监督检验检疫总局
+WH:
+  prefix: WH_culture
+  administration: 文化部
+TY:
+  prefix: TY_sports
+  administration: 国家体育总局
+SB:
+  prefix: SB_business
+  administration: 商务部
+WB:
+  prefix: WB_materials_management
+  administration: 国家发改委
+HJ:
+  prefix: HJ_environment_protection
+  administration: 国家环境保护总局
+XB:
+  prefix: XB_rare_earth
+  administration: 国家发改委稀土办公室
+CJ:
+  prefix: CJ_town_construction
+  administration: 建设部
+JG:
+  prefix: JG_construction_industry
+  administration: 建设部
+CY:
+  prefix: CY_news_publishing
+  administration: 国家新闻出版总署
+MT:
+  prefix: MT_coal
+  administration: 国家发改委
+WS:
+  prefix: WS_health
+  administration: 卫生部
+GA:
+  prefix: GA_public_safety
+  administration: 公安部
+BB:
+  prefix: BB_packaging
+  administration: 国家发改委
+DB:
+  prefix: DB_seismology
+  administration: 中国地震局
+LB:
+  prefix: LB_tourism
+  administration: 国家旅游局
+QX:
+  prefix: QX_meteorological
+  administration: 中国气象局
+WM:
+  prefix: WM_foreign_trade
+  administration: 外经贸部科技司
+HS:
+  prefix: HS_customs
+  administration: 海关总署
+YZ:
+  prefix: YZ_postal
+  administration: 国家邮政局
+GM:
+  prefix: GM_cryptography
+  administration: 国家密码管理局
+AQ:
+  prefix: AQ_production_safety
+  administration: 国家安全生产管理局
+GH:
+  prefix: GH_supply_marketing
+  administration: 中华全国供销合作总社
+LS:
+  prefix: LS_food
+  administration: 国家粮食局
+TJ:
+  prefix: TJ_railway_traffic
+  administration: 铁道部标准所
+ZY:
+  prefix: ZY_chinese_medicine
+  administration: 国家中医药管理局

data/lib/relaton/gb.rb

@@ -0,0 +1,33 @@
+require "digest/md5"
+require "mechanize"
+require "relaton/core"
+require "relaton/iso"
+require "cnccs"
+require_relative "version"
+require_relative "gb/util"
+require_relative "gb/item_data"
+require_relative "gb/item"
+require_relative "gb/bibitem"
+require_relative "gb/bibdata"
+require_relative "gb/bibliography"
+
+# if defined? Relaton
+#   require "relaton_gb/processor"
+#   # don't register the gem if it's required form relaton's registry
+#   return if caller.detect { |c| c.include? "register_gems" }
+
+#   Relaton::Registry.instance.register RelatonGb::Processor
+# end
+
+module Relaton
+  module Gb
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end