relaton-gb 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c0978ce217698027cca98e48fcad3fdf3060af8f
-  data.tar.gz: 3f9e4857d33b1b3aee204746098c003816ded415
+  metadata.gz: 64d418169f8f5f6c500a43e913ea94d13d4dca7d
+  data.tar.gz: 2717d29fdc48172660535462cade3eb856dcf362
 SHA512:
-  metadata.gz: 6a52a32b85031f0b31005a562fa3c95e630f9a33e353b7b0371eee23ebb8483430cf770bc16f900406357609ecbabae3dcb8dab1af6e2ac5a9151866bb03ceb0
-  data.tar.gz: 9d3ab095b5681268b8113789b53c0b60140e4faf427918f29561460454fec99b9cff46bb9d39d5b6480df341f7d23396c17bcb8258fa95f413965a7f136f4e96
+  metadata.gz: e372dd88445fdb97b2bda4c41b755bb8b325ade7c271f0db0e567c697c7b4c80b6d1da24c7af1e8439fee87e9e9cf1b2ff2940e1dd86d5146c1efa73ec048e4b
+  data.tar.gz: ee2e5d7edc8f836505a024bff24e543fa70a05d9508885a3ead36b798a0750aa661a825f3f33250b00740a78b5edfb02d5610d616902aa7899d3ea75046e7594

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    relaton-gb (0.6.2)
+    relaton-gb (0.6.3)
       cnccs (~> 0.1.1)
       gb-agencies (~> 0.0.1)
       relaton-iso-bib (~> 0.3.0)
@@ -9,11 +9,13 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     byebug (11.0.1)
     cnccs (0.1.3)
     coderay (1.1.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
     debase (0.2.2)
       debase-ruby_core_source (>= 0.10.2)
     debase-ruby_core_source (0.10.5)
@@ -22,6 +24,7 @@ GEM
     equivalent-xml (0.6.0)
       nokogiri (>= 1.4.3)
     gb-agencies (0.0.5)
+    hashdiff (1.0.0)
     isoics (0.1.7)
     json (2.2.0)
     method_source (0.9.2)
@@ -34,12 +37,12 @@ GEM
     pry-byebug (3.7.0)
       byebug (~> 11.0)
       pry (~> 0.10)
-    public_suffix (3.1.1)
+    public_suffix (4.0.1)
     rake (10.5.0)
-    relaton-bib (0.3.5)
+    relaton-bib (0.3.6)
       addressable
-      nokogiri (~> 1.10)
-    relaton-iso-bib (0.3.4)
+      nokogiri
+    relaton-iso-bib (0.3.5)
       isoics (~> 0.1.6)
       relaton-bib (~> 0.3.0)
       ruby_deep_clone (~> 0.8.0)
@@ -59,11 +62,17 @@ GEM
     ruby-debug-ide (0.7.0)
       rake (>= 0.8.1)
     ruby_deep_clone (0.8.0)
+    safe_yaml (1.0.5)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
+    vcr (5.0.0)
+    webmock (3.7.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
   ruby
@@ -78,6 +87,8 @@ DEPENDENCIES
   rspec (~> 3.0)
   ruby-debug-ide
   simplecov
+  vcr
+  webmock
 
 BUNDLED WITH
    2.0.1

data/lib/relaton_gb/gb_bibliography.rb

@@ -91,16 +91,11 @@ module RelatonGb
       def search_filter(code)
         # search filter needs to incorporate year
         docidrx = %r{^[^\s]+\s[\d\.-]+}
-        # corrigrx = %r{^[^\s]+\s[\d\.]+-[0-9]+/}
         warn "fetching #{code}..."
         result = search(code)
-        ret = result.select do |hit|
-          hit.title && hit.title.match(docidrx).to_s == code # &&
-            # !corrigrx =~ hit.title
+        result.select do |hit|
+          hit.docref && hit.docref.match(docidrx).to_s.include?(code)
         end
-        return ret unless ret.empty?
-
-        []
       end
 
       # Sort through the results from Isobib, fetching them three at a time,
@@ -125,12 +120,15 @@ module RelatonGb
         { years: missed_years }
       end
 
-      def fetch_pages(s, n)
-        workers = RelatonBib::WorkersPool.new n
+      # @param hits [RelatonBib::HitCollection<RelatonBib::Hit>]
+      # @param threads [Integer]
+      # @return [Array<RelatonBib::GbBibliographicItem>]
+      def fetch_pages(hits, threads)
+        workers = RelatonBib::WorkersPool.new threads
         workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
-        s.each_with_index { |hit, i| workers << { i: i, hit: hit } }
+        hits.each_with_index { |hit, i| workers << { i: i, hit: hit } }
         workers.end
-        workers.result.sort { |x, y| x[:i] <=> y[:i] }.map { |x| x[:hit] }
+        workers.result.sort_by { |x| x[:i] }.map { |x| x[:hit] }
       end
     end
   end

data/lib/relaton_gb/gb_scrapper.rb

@@ -16,35 +16,40 @@ module RelatonGb
       # @return [RelatonGb::HitCollection]
       def scrape_page(text)
         search_html = OpenURI.open_uri(
-          "http://www.std.gov.cn/search/stdPage?q=" + text
+          "http://openstd.samr.gov.cn/bzgk/gb/std_list?p.p2=" + text
         )
         result = Nokogiri::HTML search_html
-        hits = result.css(".s-title a").map do |h|
-          Hit.new pid: h[:pid], title: h.text, scrapper: self
+        hits = result.xpath(
+          "//table[contains(@class, 'result_list')]/tbody[2]/tr",
+        ).map do |h|
+          ref = h.at "./td[2]/a"
+          pid = ref[:onclick].match(/[0-9A-F]+/).to_s
+          rdate = h.at("./td[7]").text
+          Hit.new pid: pid, docref: ref.text, scrapper: self, release_date: rdate
         end
-        HitCollection.new hits
+        HitCollection.new hits.sort_by(&:release_date).reverse
       rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError
-        raise RelatonBib::RequestError, "Cannot access http://www.std.gov.cn/search/stdPage"
+        raise RelatonBib::RequestError, "Cannot access http://www.std.gov.cn/bzgk/gb/std_list"
       end
 
-      # @param pid [Strin] standard's page id
+      # @param hit [RelatonGb::Hit] standard's page id
       # @return [RelatonGb::GbBibliographicItem]
-      def scrape_doc(pid)
-        src = "http://www.std.gov.cn/gb/search/gbDetailed?id=" + pid
+      def scrape_doc(hit)
+        src = "http://openstd.samr.gov.cn/bzgk/gb/newGbInfo?hcno=" + hit.pid
         doc = Nokogiri::HTML OpenURI.open_uri(src)
-        GbBibliographicItem.new scrapped_data(doc, src: src)
+        GbBibliographicItem.new scrapped_data(doc, src, hit)
       rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError
         raise RelatonBib::RequestError, "Cannot access #{src}"
       end
 
       # @param doc [Nokogiri::HTML]
+      # @param _ref [String]
       # @return [Hash]
       #   * :type [String]
       #   * :name [String]
-      def get_committee(doc)
-        name = doc.xpath("//p/a[1]/following-sibling::text()").text.
-          match(/(?<=（)[^）]+/).to_s
-        { type: "technical", name: name }
+      def get_committee(doc, _ref)
+        name = doc.at("//div[contains(text(), '归口单位')]/following-sibling::div")
+        { type: "technical", name: name.text.delete("\r\n\t\t") }
       end
     end
   end

data/lib/relaton_gb/hit.rb

@@ -7,28 +7,36 @@ module RelatonGb
     attr_reader :hit_collection
 
     # @return [String]
-    attr_reader :pid
+    attr_reader :pid, :docref
 
-    # @return [String]
-    attr_reader :title
+    # @return [Date, NilClass]
+    attr_reader :release_date
+
+    # @return [String, NilClass]
+    attr_reader :status
 
     # @return [RelatonGb::GbScrapper, RelatonGb::SecScraper, RelatonGb::TScrapper]
     attr_reader :scrapper
 
-    # @param hit [Hash]
-    # @param hit_collection [Isobib:HitCollection]
-    def initialize(pid:, title:, hit_collection: nil, scrapper:)
+    # @param pid [String]
+    # @param docref [String]
+    # @parma scrapper [RelatonGb::GbScrapper, RelatonGb::SecScraper, RelatonGb::TScrapper]
+    # @param release_date [String]
+    # @status [String, NilClass]
+    # @param hit_collection [RelatonGb:HitCollection, NilClass]
+    def initialize(pid:, docref:, scrapper:, **args)
       @pid            = pid
-      @title          = title
-      @hit_collection = hit_collection
+      @docref         = docref
       @scrapper       = scrapper
-      self.hit_collection << self if hit_collection
+      @release_date   = Date.parse args[:release_date] if args[:release_date]
+      @status         = args[:status]
+      @hit_collection = args[:hit_collection]
     end
 
     # Parse page.
     # @return [Isobib::IsoBibliographicItem]
     def fetch
-      @fetch ||= scrapper.scrape_doc pid
+      @fetch ||= scrapper.scrape_doc self
     end
 
     # @return [String]
@@ -40,21 +48,7 @@ module RelatonGb
     def inspect
       "<#{self.class}:#{format('%#.14x', object_id << 1)} "\
       "@fullIdentifier=\"#{@fetch&.shortref}\" "\
-      "@title=\"#{title}\">"
+      "@docref=\"#{docref}\">"
     end
-
-    # @param builder [Nokogiri::XML::Builder]
-    # @param opts [Hash]
-    # @return [String]
-    # def to_xml(builder = nil, opts = {})
-    #   if builder
-    #     fetch.to_xml builder, opts
-    #   else
-    #     builder = Nokogiri::XML::Builder.new(encoding: "UTF-8") do |xml|
-    #       fetch.to_xml xml, opts
-    #     end
-    #     builder.doc.root.to_xml
-    #   end
-    # end
   end
 end

data/lib/relaton_gb/processor.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "relaton/processor"
+
+module RelatonGb
+  class Processor < Relaton::Processor
+    def initialize
+      @short = :relaton_gb
+      @prefix = "CN"
+      @defaultprefix = %r{^GB }
+      @idtype = "Chinese Standard"
+    end
+
+    # @param code [String]
+    # @param date [String, NilClass] year
+    # @param opts [Hash]
+    # @return [RelatonGb::GbBibliographicItem]
+    def get(code, date, opts)
+      ::RelatonGb::GbBibliography.get(code, date, opts)
+    end
+
+    # @param xml [String]
+    # @return [RelatonGb::GbBibliographicItem]
+    def from_xml(xml)
+      ::RelatonGb::XMLParser.from_xml xml
+    end
+
+    # @param hash [Hash]
+    # @return [RelatonGb::GbBibliographicItem]
+    def hash_to_bib(hash)
+      item_hash = ::RelatonGb::HashConverter.hash_to_bib(hash)
+      ::RelatonGb::GbBibliographicItem.new item_hash
+    end
+  end
+end

data/lib/relaton_gb/scrapper.rb

@@ -11,64 +11,54 @@ module RelatonGb
 
     # rubocop:disable Metrics/MethodLength
     # @param doc [Nokogiri::HTML::Document]
-    # @param src [String] url of scrapped page
+    # @param src [String]
+    # @param hit [RelatonGb::Hit]
     # @return [Hash]
-    def scrapped_data(doc, src:)
+    def scrapped_data(doc, src, hit)
       {
-        committee: get_committee(doc),
-        docid: get_docid(doc),
+        committee: get_committee(doc, hit.docref),
+        docid: get_docid(hit.docref),
         title: get_titles(doc),
-        contributor: get_contributors(doc),
-        type: get_type(doc),
-        docstatus: get_status(doc),
-        gbtype: get_gbtype(doc),
+        contributor: get_contributors(doc, hit.docref),
+        type: get_type,
+        docstatus: get_status(doc, hit.status),
+        gbtype: get_gbtype(doc, hit.docref),
         ccs: get_ccs(doc),
         ics: get_ics(doc),
         link: [{ type: "src", content: src }],
         date: get_dates(doc),
         language: ["zh"],
         script: ["Hans"],
-        structuredidentifier: fetch_structuredidentifier(doc),
+        structuredidentifier: fetch_structuredidentifier(hit.docref),
       }
     end
     # rubocop:enable Metrics/MethodLength
 
-    # @param doc [Nokogiri::HTML::Document]
-    # @param xpt [String]
+    # @param docref [String]
     # @return [Array<RelatonBib::DocumentIdentifier>]
-    def get_docid(doc, xpt = '//dt[text()="标准号"]/following-sibling::dd[1]')
-      item_ref = doc.at xpt
-      return [] unless item_ref
-
-      [RelatonBib::DocumentIdentifier.new(id: item_ref.text, type: "Chinese Standard")]
+    def get_docid(docref)
+      [RelatonBib::DocumentIdentifier.new(id: docref, type: "Chinese Standard")]
     end
 
-    # @param doc [Nokogiri::HTML::Document]
-    # @param xpt [String]
+    # @param docref [String]
     # @return [RelatonIsoBib::StructuredIdentifier]
-    def fetch_structuredidentifier(doc, xpt = '//dt[text()="标准号"]/following-sibling::dd[1]')
-      item_ref = doc.at xpt
-      unless item_ref
-        return RelatonIsoBib::StructuredIdentifier.new(
-          project_number: "?", part_number: "?", prefix: nil, id: "?",
-          type: "Chinese Standard"
-        )
-      end
-
-      m = item_ref.text.match(/^([^–—.-]*\d+)\.?((?<=\.)\d+|)/)
-      # prefix = doc.xpath(xpt).text.match(/^[^\s]+/).to_s
+    def fetch_structuredidentifier(docref)
+      m = docref.match(/^([^–—.-]*\d+)\.?((?<=\.)\d+|)/)
       RelatonIsoBib::StructuredIdentifier.new(
         project_number: m[1], part_number: m[2], prefix: nil,
-        id: item_ref.text, type: "Chinese Standard"
+        id: docref, type: "Chinese Standard"
       )
     end
 
-    def get_contributors(doc, xpt = '//dt[text()="标准号"]/following-sibling::dd[1]')
+    # @param doc [Nokogiri::HTML::Document]
+    # @param docref [Strings]
+    # @return [Array<Hash>]
+    def get_contributors(doc, docref)
       gb_en = GbAgencies::Agencies.new("en", {}, "")
       gb_zh = GbAgencies::Agencies.new("zh", {}, "")
-      name = doc.xpath(xpt).text.match(/^[^\s]+/).to_s
+      name = docref.match(/^[^\s]+/).to_s
       name.sub!(%r{/[TZ]$}, "") unless name =~ /^GB/
-      gbtype = get_gbtype(doc)
+      gbtype = get_gbtype(doc, docref)
       entity = RelatonBib::Organization.new name: [
         { language: "en", content: gb_en.standard_agency1(gbtype[:scope], name, gbtype[:mandate]) },
         { language: "zh", content: gb_zh.standard_agency1(gbtype[:scope], name, gbtype[:mandate]) },
@@ -83,57 +73,56 @@ module RelatonGb
     #   * :language [String]
     #   * :script [String]
     def get_titles(doc)
-      titles = [{ title_main: doc.css("div.page-header h4").text, title_intro: nil,
-                  language: "zh", script: "Hans" }]
-      title_main = doc.css("div.page-header h5").text
+      titles = [{ title_main: doc.at("//td[contains(text(), '中文标准名称')]/b").text,
+                  title_intro: nil, language: "zh", script: "Hans" }]
+      title_main = doc.at("//td[contains(text(), '英文标准名称')]").text.match(/[\w\s]+/).to_s
       unless title_main.empty?
         titles << { title_main: title_main, title_intro: nil, language: "en", script: "Latn" }
       end
       titles
     end
 
-    def get_type(_doc)
-      "international-standard"
+    def get_type
+      "standard"
     end
 
     # @param doc [Nokogiri::HTML::Document]
-    # @param xpt [String]
+    # @param status [String, NilClass]
     # @return [RelatonBib::DocumentStatus]
-    def get_status(doc, xpt = ".s-status.label:nth-child(3)")
-      case doc.at(xpt).text.gsub(/\s/, "")
-      when "即将实施"
-        stage = "published"
-      when "现行"
-        stage = "activated"
-      when "废止"
-        stage = "obsoleted"
-      end
+    def get_status(doc, status = nil)
+      stage = case status || doc.at("//td[contains(., '标准状态')]/span")&.text
+              when "即将实施" then "published"
+              when "现行" then "activated"
+              when "废止" then "obsoleted"
+              end
       RelatonBib::DocumentStatus.new stage: stage
     end
 
     private
 
     # @param doc [Nokogiri::HTML::Document]
+    # @param ref [String]
     # @return [Hash]
     #   * :scope [String]
     #   * :prefix [String]
     #   * :mandate [String]
-    def get_gbtype(doc)
-      ref = get_ref(doc)
+    def get_gbtype(doc, ref)
+      # ref = get_ref(doc)
       { scope: get_scope(doc), prefix: get_prefix(ref)["prefix"],
         mandate: get_mandate(ref) }
     end
 
     # @param doc [Nokogiri::HTML::Document]
     # @return [String]
-    def get_ref(doc)
-      doc.xpath('//dt[text()="标准号"]/following-sibling::dd[1]').text
-    end
+    # def get_ref(doc)
+    #   doc.xpath('//dt[text()="标准号"]/following-sibling::dd[1]').text
+    # end
 
     # @param doc [Nokogiri::HTML::Document]
     # @return [Array<String>]
     def get_ccs(doc)
-      [doc&.xpath('//dt[text()="中国标准分类号"]/following-sibling::dd[1]')&.text]
+      [doc.at("//div[contains(text(), '中国标准分类号')]/following-sibling::div").
+        text.delete("\r\n\t\t")]
     end
 
     # @param doc [Nokogiri::HTML::Document]
@@ -142,21 +131,21 @@ module RelatonGb
     #   * :group [String]
     #   * :subgroup [String]
     def get_ics(doc)
-      ics = doc.xpath('//dt[(.="国际标准分类号")]/following-sibling::dd[1]/span')
-      return [] if ics.empty?
+      ics = doc.at("//div[contains(text(), '国际标准分类号')]/following-sibling::div"\
+                   " | //dt[contains(text(), '国际标准分类号')]/following-sibling::dd")
+      return [] unless ics
 
-      field, group, subgroup = ics.text.split "."
+      field, group, subgroup = ics.text.delete("\r\n\t\t").split "."
       [{ field: field, group: group.ljust(3, "0"), subgroup: subgroup }]
     end
 
     # @param doc [Nokogiri::HTML::Document]
     # @return [String]
     def get_scope(doc)
-      scope = doc.at(".s-status.label-info").text
-      if scope == "国家标准"
-        "national"
-      elsif scope =~ /^行业标准/
-        "sector"
+      issued = doc.at("//div[contains(., '发布单位')]/following-sibling::div")
+      case issued&.text
+      when /国家标准/ then "national"
+      when /^行业标准/ then "sector"
       end
     end
 
@@ -170,8 +159,7 @@ module RelatonGb
     # @param pref [String]
     # @return [Hash{String=>String}]
     def prefix(pref)
-      file_path = File.join(__dir__, "yaml/prefixes.yaml")
-      @prefixes ||= YAML.load_file(file_path)
+      @prefixes ||= YAML.load_file File.join(__dir__, "yaml/prefixes.yaml")
       @prefixes[pref]
     end
 
@@ -190,8 +178,9 @@ module RelatonGb
     #   * :type [String] type of date
     #   * :on [String] date
     def get_dates(doc)
-      date = doc.xpath('//dt[.="发布日期"]/following-sibling::dd[1]').text
-      [{ type: "published", on: date }]
+      date = doc.at("//div[contains(text(), '发布日期')]/following-sibling::div"\
+                    " | //dt[contains(text(), '发布日期')]/following-sibling::dd")
+      [{ type: "published", on: date.text.delete("\r\n\t\t") }]
     end
   end
 end

data/lib/relaton_gb/sec_scrapper.rb

@@ -18,42 +18,77 @@ module RelatonGb
       # @param text [String] code of standard for serarch
       # @return [RelatonGb::HitCollection]
       def scrape_page(text)
-        uri = URI "http://www.std.gov.cn/hb/search/hbPage?searchText=#{text}"
-        res = JSON.parse Net::HTTP.get(uri)
-        hits = res["rows"].map do |r|
-          Hit.new pid: r["id"], title: r["STD_CODE"], scrapper: self
+        # uri = URI "http://www.std.gov.cn/hb/search/hbPage?searchText=#{text}"
+        uri = URI "http://hbba.sacinfo.org.cn/stdQueryList"
+        resp = Net::HTTP.post uri, URI.encode_www_form({ key: text })
+        # res = JSON.parse Net::HTTP.get(uri)
+        json = JSON.parse resp.body
+        hits = json["records"].map do |h|
+          Hit.new pid: h["pk"], docref: h["code"], status: h["status"], scrapper: self
         end
+        # hits = res["rows"].map do |r|
+        #   Hit.new pid: r["id"], title: r["STD_CODE"], scrapper: self
+        # end
         HitCollection.new hits
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
              Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
-             OpenSSL::SSL::SSLError
+             OpenSSL::SSL::SSLError, Errno::ETIMEDOUT
         raise RelatonBib::RequestError, "Cannot access #{uri}"
       end
 
-      # @param pid [String] standard's page id
+      # @param hit [RelatonGb::Hit]
       # @return [RelatonGb::GbBibliographicItem]
-      def scrape_doc(pid)
-        src = "http://www.std.gov.cn/hb/search/stdHBDetailed?id=#{pid}"
+      def scrape_doc(hit)
+        src = "http://hbba.sacinfo.org.cn/stdDetail/#{hit.pid}"
         page_uri = URI src
         doc = Nokogiri::HTML Net::HTTP.get(page_uri)
-        GbBibliographicItem.new scrapped_data(doc, src: src)
+        GbBibliographicItem.new scrapped_data(doc, src, hit)
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
              Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
-             OpenSSL::SSL::SSLError
+             OpenSSL::SSL::SSLError, Errno::ETIMEDOUT
         raise RelatonBib::RequestError, "Cannot access #{src}"
       end
 
       private
 
       # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<Hash>]
+      #   * :title_intro [String]
+      #   * :title_main [String]
+      #   * :language [String]
+      #   * :script [String]
+      def get_titles(doc)
+        titles = [{ title_main: doc.at("//h4").text.delete("\r\n\t"),
+                    title_intro: nil, language: "zh", script: "Hans" }]
+        # title_main = doc.at("//td[contains(text(), '英文标准名称')]").text.match(/[\w\s]+/).to_s
+        # unless title_main.empty?
+        #   titles << { title_main: title_main, title_intro: nil, language: "en", script: "Latn" }
+        # end
+        titles
+      end
+
+      # @param _doc [Nokogiri::HTML::Document]
+      # @param ref [String]
       # @return [Hash]
       #   * :type [String]
       #   * :name [String]
-      def get_committee(doc)
-        ref = get_ref(doc)
+      def get_committee(_doc, ref)
+        # ref = get_ref(doc)
         name = get_prefix(ref)["administration"]
         { type: "technical", name: name }
       end
+
+      # @param _doc [Nokogiri::HTML::Document]
+      # @return [String]
+      def get_scope(_doc)
+        "sector"
+      end
+
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<String>]
+      def get_ccs(doc)
+        [doc.at("//dt[contains(text(), '中国标准分类号')]/following-sibling::dd").text]
+      end
     end
   end
 end

data/lib/relaton_gb/t_scrapper.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "open-uri"
+require "net/http"
 require "nokogiri"
 require "relaton_gb/scrapper"
 require "relaton_gb/gb_bibliographic_item"
@@ -21,13 +22,15 @@ module RelatonGb
         search_html = OpenURI.open_uri(
           "http://www.ttbz.org.cn/Home/Standard?searchType=2&key=" +
           CGI.escape(text.tr("-", [8212].pack("U"))),
-        )
+        ).read
         header = Nokogiri::HTML search_html
         xpath = '//table[contains(@class, "standard_list_table")]/tr/td/a'
-        t_xpath = "../preceding-sibling::td[3]"
+        t_xpath = "../preceding-sibling::td[4]"
         hits = header.xpath(xpath).map do |h|
-          title = h.at(t_xpath).text.gsub(/â\u0080\u0094/, "-")
-          Hit.new pid: h[:href].sub(%r{\/$}, ""), title: title, scrapper: self
+          docref = h.at(t_xpath).text.gsub(/â\u0080\u0094/, "-")
+          status = h.at("../preceding-sibling::td[1]").text.delete "\r\n"
+          pid = h[:href].sub(%r{\/$}, "")
+          Hit.new pid: pid, docref: docref, status: status, scrapper: self
         end
         HitCollection.new hits
       rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError
@@ -35,12 +38,12 @@ module RelatonGb
       end
       # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
-      # @param pid [String] standard's page path
+      # @param hit [RelatonGb::Hit] standard's page path
       # @return [RelatonGb::GbBibliographicItem]
-      def scrape_doc(pid)
-        src = "http://www.ttbz.org.cn#{pid}"
+      def scrape_doc(hit)
+        src = "http://www.ttbz.org.cn#{hit.pid}"
         doc = Nokogiri::HTML OpenURI.open_uri(src), nil, Encoding::UTF_8.to_s
-        GbBibliographicItem.new scrapped_data(doc, src: src)
+        GbBibliographicItem.new scrapped_data(doc, src, hit)
       rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError
         raise RelatonBib::RequestError, "Cannot access #{src}"
       end
@@ -49,16 +52,18 @@ module RelatonGb
 
       # rubocop:disable Metrics/MethodLength
       # @param doc [Nokogiri::HTML::Document]
+      # @param src [String]
+      # @param hit [RelatonGb::Hit]
       # @return [Hash]
-      def scrapped_data(doc, src:)
-        docid_xpt  = '//td[contains(.,"标准编号")]/following-sibling::td[1]'
-        status_xpt = '//td[contains(.,"标准状态")]/following-sibling::td[1]/span'
+      def scrapped_data(doc, src, hit)
+        # docid_xpt  = '//td[contains(.,"标准编号")]/following-sibling::td[1]'
+        # status_xpt = '//td[contains(.,"标准状态")]/following-sibling::td[1]/span'
         {
-          committee: get_committee(doc),
-          docid: get_docid(doc, docid_xpt),
+          committee: get_committee(doc, hit.docref),
+          docid: get_docid(hit.docref),
           title: get_titles(doc),
-          type: "international-standard",
-          docstatus: get_status(doc, status_xpt),
+          type: get_type,
+          docstatus: get_status(doc, hit.status),
           gbtype: gbtype,
           ccs: get_ccs(doc),
           ics: get_ics(doc),
@@ -66,12 +71,12 @@ module RelatonGb
           date: get_dates(doc),
           language: ["zh"],
           script: ["Hans"],
-          structuredidentifier: fetch_structuredidentifier(doc),
+          structuredidentifier: fetch_structuredidentifier(hit.docref),
         }
       end
       # rubocop:enable Metrics/MethodLength
 
-      def get_committee(doc)
+      def get_committee(doc, _ref)
         {
           name: doc.xpath('//td[.="团体名称"]/following-sibling::td[1]').text,
           type: "technical",

data/lib/relaton_gb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RelatonGb
-  VERSION = "0.6.2"
+  VERSION = "0.6.3"
 end

data/lib/relaton_gb.rb

@@ -1,7 +1,10 @@
 require "relaton_gb/version"
 require "relaton_gb/gb_bibliography"
 
-if defined? Relaton
-  require_relative "relaton/processor"
-  Relaton::Registry.instance.register Relaton::RelatonGb::Processor
-end
+# if defined? Relaton
+#   require "relaton_gb/processor"
+#   # don't register the gem if it's required form relaton's registry
+#   return if caller.detect { |c| c.include? "register_gems" }
+
+#   Relaton::Registry.instance.register RelatonGb::Processor
+# end

data/relaton_gb.gemspec

@@ -32,6 +32,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "ruby-debug-ide"
   spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "vcr"
+  spec.add_development_dependency "webmock"
 
   spec.add_dependency "cnccs", "~> 0.1.1"
   spec.add_dependency "gb-agencies", "~> 0.0.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-gb
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-20 00:00:00.000000000 Z
+date: 2019-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,6 +122,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: cnccs
   requirement: !ruby/object:Gem::Requirement
@@ -185,7 +213,6 @@ files:
 - appveyor.yml
 - bin/console
 - bin/setup
-- lib/relaton/processor.rb
 - lib/relaton_gb.rb
 - lib/relaton_gb/ccs.rb
 - lib/relaton_gb/gb_bibliographic_item.rb
@@ -196,6 +223,7 @@ files:
 - lib/relaton_gb/hash_converter.rb
 - lib/relaton_gb/hit.rb
 - lib/relaton_gb/hit_collection.rb
+- lib/relaton_gb/processor.rb
 - lib/relaton_gb/scrapper.rb
 - lib/relaton_gb/sec_scrapper.rb
 - lib/relaton_gb/t_scrapper.rb

data/lib/relaton/processor.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-require "relaton/processor"
-
-module Relaton
-  module RelatonGb
-    class Processor < Relaton::Processor
-      def initialize
-        @short = :relaton_gb
-        @prefix = "CN"
-        @defaultprefix = %r{^GB }
-        @idtype = "Chinese Standard"
-      end
-
-      def get(code, date, opts)
-        ::RelatonGb::GbBibliography.get(code, date, opts)
-      end
-
-      def from_xml(xml)
-        ::RelatonGb::XMLParser.from_xml xml
-      end
-    end
-  end
-end