relaton-cie 1.9.0 → 1.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a32430b5610333200d49320b428edb59cdd4d05c12c11aa6be6320c88058fe81
-  data.tar.gz: 938409879b49ad789a422391356e4548413735dd2a1154c6203425b6e6575eb4
+  metadata.gz: 2d5d26de612f4a997641b1f7325f48624e0f9d349d43373236bb10ec27115fb8
+  data.tar.gz: 31bec1cd855722cff7d70b57106b78182d98683c8c98984027827cf1f8ae9579
 SHA512:
-  metadata.gz: 5e143606dc83ddd5a7c128edb77e0264127accfe466007711865517e6212c9aae9c95892ccc5b170743dfba9d7a37879c504ed6e4e4293aa75498c9cfdae281e
-  data.tar.gz: 85c9aacec26c847278d060979ba7ac56619817f02065f890783fdcc0147b0684cb2ea905757270470b9048e8dca5f9ddc2b6aa3e4771f130b1c5b358401f21ea
+  metadata.gz: 63027b7c118820397e616d083cd6aed0748fc9eb13efa3d12428a7441bddfb9f31b9375825cea31825cccc3ba518c9ffc39c0e4984299cdf49c519c5dc1d979a
+  data.tar.gz: e01df911e2fe2d1974bfc5004bbbf49ffb336872c7bead4722997050e38c141ef349d41f5c92f31d7acebdaa4c02dcca1cde012983c79533255687e77f31d298

data/Gemfile

@@ -3,5 +3,5 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in relaton_cie.gemspec
 gemspec
 
-gem "rake", "~> 12.0"
+gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/lib/relaton_cie/data_fetcher.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+require "English"
+require "fileutils"
+require "mechanize"
+require "relaton_bib"
+
+module RelatonCie
+  class DataFetcher
+    URL = "https://www.techstreet.com/cie/searches/31156444?page=1&per_page=100"
+
+    def initialize(output, format)
+      @agent = Mechanize.new
+      @output = output
+      @format = format
+    end
+
+    # @param hit [Nokogiri::HTML::Document]
+    # @param doc [Mechanize::Page]
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    def fetch_docid(hit, doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      code = hit.at("h3/a").text.strip.sub(/\u25b9/, "").gsub(" / ", "/")
+      c2idx = %r{(?:\(|/)(?<c2>(?:ISO|IEC)\s[^()]+)} =~ code
+      code = code[0...c2idx].strip if c2idx
+      /^(?<code1>[^(]+)(?:\((?<code2>\w+\d+,(?:\sPages)?[^)]+))?/ =~ code
+      if code1.match?(/^CIE/)
+        c = code1.size > 25 && code2 ? "CIE #{code2.sub(/,(\sPages)?/, '')}" : code1
+        add = doc.at("//hgroup/h2")&.text&.match(/(Add)endum\s(\d+)$/)
+        c += " #{add[1]} #{add[2]}" if add
+      elsif (pcode = doc.at('//dt[.="Product Code(s):"]/following-sibling::dd'))
+        c = "CIE #{pcode.text.strip.match(/[^,]+/)}"
+      else
+        num = code.match(/(?<=\()\w{2}\d+,.+(?=\))/).to_s.gsub(/,(?=\s)/, "")
+          .gsub(/,(?=\S)/, " ")
+        c = "CIE #{num}"
+      end
+      docid = [RelatonBib::DocumentIdentifier.new(type: "CIE", id: c)]
+      isbn = doc.at('//dt[contains(.,"ISBN")]/following-sibling::dd')
+      docid << RelatonBib::DocumentIdentifier.new(type: c2.match(/\w+/).to_s, id: c2.strip) if c2
+      docid << RelatonBib::DocumentIdentifier.new(type: "ISBN", id: isbn.text.strip) if isbn
+      docid
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [RelatonBib::TypedTitleStringCollection, Array]
+    def fetch_title(doc)
+      t = doc.at("//hgroup/h2", "//hgroup/h1")
+      return [] unless t
+
+      RelatonBib::TypedTitleString.from_string t.text.strip
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Array<RelatonBib::BibliographicDate>]
+    def fetch_date(doc)
+      doc.xpath('//dt[.="Published:"]/following-sibling::dd[1]').map do |d|
+        pd = d.text.strip
+        on = pd.match?(/^\d{4}(?:[^-]|$)/) ? pd : Date.strptime(pd, "%m/%d/%Y").strftime("%Y-%m-%d")
+        RelatonBib::BibliographicDate.new(type: "published", on: on)
+      end
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [String]
+    def fetch_edition(doc)
+      doc.at('//dt[.="Edition:"]/following-sibling::dd')&.text&.match(/^\d+(?=th)/)&.to_s
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Array<Hash>]
+    def fetch_relation(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      doc.xpath('//section[@class="history"]/ol/li[not(contains(@class,"selected-product"))]').map do |rel|
+        ref = rel.at("a")
+        url = "https://www.techstreet.com#{ref[:href]}"
+        title = RelatonBib::TypedTitleString.from_string ref.at('p/span[@class="title"]').text
+        did = ref.at("h3").text
+        docid = [RelatonBib::DocumentIdentifier.new(type: "CIE", id: did)]
+        on = ref.at("p/time")
+        date = [RelatonBib::BibliographicDate.new(type: "published", on: on[:datetime])]
+        link = [RelatonBib::TypedUri.new(type: "src", content: url)]
+        bibitem = RelatonBib::BibliographicItem.new docid: docid, title: title, link: link, date: date
+        type = ref.at('//li/i[contains(@class,"historical")]') ? "updates" : "updatedBy"
+        { type: type, bibitem: bibitem }
+      end
+    end
+
+    # @param url [String]
+    # @return [Array<RelatonBib::TypedUri>]
+    def fetch_link(url)
+      [RelatonBib::TypedUri.new(type: "src", content: url)]
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Array<RelatonBib::FormattedString>]
+    def fetch_abstract(doc)
+      content = doc.at('//div[contains(@class,"description")]')&.text&.strip
+      return [] if content.nil? || content.empty?
+
+      [RelatonBib::FormattedString.new(content: content, language: "en",
+                                       script: "Latn")]
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Array<Hash>]
+    def fetch_contributor(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity
+      authors = doc.xpath('//hgroup/p[not(@class="pub_date")]').text
+      contribs = []
+      until authors.empty?
+        /^(?<sname1>\S+(?:\sder?\s)?[^\s,]+)
+        (?:,?\s(?<sname2>[\w-]{2,})(?=,\s+\w\.))?
+        (?:,?\s(?<fname>[\w-]{2,})(?!,\s+\w\.))?
+        (?:(?:\s?,\s?|\s)(?<init>(?:\w(?:\s?\.|\s|,|$)[\s-]?)+))?
+        (?:(?:,\s?|\s|\.|(?<=\s))(?:and\s)?)?/x =~ authors
+        raise StandardError, "Author name not found in \"#{authors}\"" unless $LAST_MATCH_INFO
+
+        authors.sub! $LAST_MATCH_INFO.to_s, ""
+        sname = [sname1, sname2].compact.join " "
+        surname = RelatonBib::LocalizedString.new sname, "en", "Latn"
+        initial = (init&.strip || "").split(/(?:,|\.)(?:-|\s)?/).map do |int|
+          RelatonBib::LocalizedString.new(int.strip, "en", "Latn")
+        end
+        forename = fname ? [RelatonBib::LocalizedString.new(fname, "en", "Latn")] : []
+        fullname = RelatonBib::FullName.new surname: surname, forename: forename, initial: initial
+        person = RelatonBib::Person.new name: fullname
+        contribs << { entity: person, role: [{ type: "author" }] }
+      end
+      org = RelatonBib::Organization.new(
+        name: "Commission Internationale de L'Eclairage", abbreviation: "CIE",
+        url: "cie.co.at"
+      )
+      contribs << { entity: org, role: [{ type: "publisher" }] }
+    end
+
+    # @param bib [RelatonItu::ItuBibliographicItem]
+    def write_file(bib)
+      id = bib.docidentifier[0].id.gsub(%r{[/\s\-:.]}, "_")
+      file = "#{@output}/#{id.upcase}.#{@format}"
+      # if File.exist? file
+      #   warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
+      #   warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
+      # else
+      out = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
+      File.write file, out, encoding: "UTF-8"
+      # end
+    end
+
+    # @param hit [Nokogiri::HTML::Element]
+    def parse_page(hit) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      url = "https://www.techstreet.com#{hit.at('h3/a')[:href]}"
+      doc = time_req { @agent.get url }
+      item = RelatonBib::BibliographicItem.new(
+        type: "standard", docid: fetch_docid(hit, doc), title: fetch_title(doc),
+        link: fetch_link(url), abstract: fetch_abstract(doc),
+        date: fetch_date(doc), edition: fetch_edition(doc),
+        contributor: fetch_contributor(doc), relation: fetch_relation(doc),
+        language: ["en"], script: ["Latn"], doctype: "document"
+      )
+      write_file item
+    rescue StandardError => e
+      warn "Document: #{url}"
+      warn e.message
+      warn e.backtrace
+    end
+
+    def fetch(url)
+      result = time_req { @agent.get url }
+      result.xpath("//li[@data-product]").each { |hit| parse_page hit }
+      np = result.at '//a[@class="next_page"]'
+      fetch "https://www.techstreet.com#{np[:href]}" if np
+    end
+
+    def time_req
+      t1 = Time.now
+      result = yield
+      t = 1 - (Time.now - t1)
+      sleep t if t.positive?
+      result
+    end
+
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+
+      FileUtils.mkdir output unless Dir.exist? output
+      new(output, format).fetch URL
+
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+  end
+end

data/lib/relaton_cie/processor.rb

@@ -2,11 +2,12 @@ require "relaton/processor"
 
 module RelatonCie
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_cie
       @prefix = "CIE"
       @defaultprefix = /^CIE(-|\s)/
       @idtype = "CIE"
+      @datasets = %w[cie-techstreet]
     end
 
     # @param code [String]
@@ -17,6 +18,18 @@ module RelatonCie
       ::RelatonCie::CieBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the docukents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonBib::BibliographicItem]
     def from_xml(xml)

data/lib/relaton_cie/version.rb

@@ -1,3 +1,3 @@
 module RelatonCie
-  VERSION = "1.9.0".freeze
+  VERSION = "1.9.1".freeze
 end

data/lib/relaton_cie.rb

@@ -4,6 +4,7 @@ require "relaton_bib"
 require "relaton_cie/version"
 require "relaton_cie/cie_bibliography"
 require "relaton_cie/scrapper"
+require "relaton_cie/data_fetcher"
 
 module RelatonCie
   # Returns hash of XML reammar

data/relaton_cie.gemspec

@@ -31,11 +31,11 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
   spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "ruby-jing"
   spec.add_development_dependency "simplecov"
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
 
+  spec.add_dependency "mechanize", "~> 2.8.0"
   spec.add_dependency "relaton-bib", "~> 1.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-cie
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.9.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-26 00:00:00.000000000 Z
+date: 2021-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -38,20 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
 - !ruby/object:Gem::Dependency
   name: ruby-jing
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
@@ -139,6 +139,7 @@ files:
 - README.adoc
 - Rakefile
 - bin/console
+- bin/rspec
 - bin/setup
 - grammars/basicdoc.rng
 - grammars/biblio.rng
@@ -146,6 +147,7 @@ files:
 - grammars/reqt.rng
 - lib/relaton_cie.rb
 - lib/relaton_cie/cie_bibliography.rb
+- lib/relaton_cie/data_fetcher.rb
 - lib/relaton_cie/processor.rb
 - lib/relaton_cie/scrapper.rb
 - lib/relaton_cie/version.rb