relaton-cie 1.7.pre1 → 1.9.1

data/grammars/reqt.rng

@@ -30,15 +30,34 @@
         <data type="boolean"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="number"/>
+    </optional>
     <optional>
       <attribute name="subsequence"/>
     </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
     <attribute name="id">
       <data type="ID"/>
     </attribute>
     <optional>
       <attribute name="filename"/>
     </optional>
+    <optional>
+      <attribute name="model"/>
+    </optional>
+    <optional>
+      <attribute name="type"/>
+    </optional>
     <optional>
       <ref name="reqtitle"/>
     </optional>
@@ -48,9 +67,9 @@
     <optional>
       <ref name="subject"/>
     </optional>
-    <optional>
+    <zeroOrMore>
       <ref name="reqinherit"/>
-    </optional>
+    </zeroOrMore>
     <zeroOrMore>
       <ref name="classification"/>
     </zeroOrMore>
@@ -135,6 +154,16 @@
         <data type="boolean"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
     <oneOrMore>
       <ref name="BasicBlock"/>
     </oneOrMore>

data/lib/relaton_cie/data_fetcher.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+require "English"
+require "fileutils"
+require "mechanize"
+require "relaton_bib"
+
+module RelatonCie
+  class DataFetcher
+    URL = "https://www.techstreet.com/cie/searches/31156444?page=1&per_page=100"
+
+    def initialize(output, format)
+      @agent = Mechanize.new
+      @output = output
+      @format = format
+    end
+
+    # @param hit [Nokogiri::HTML::Document]
+    # @param doc [Mechanize::Page]
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    def fetch_docid(hit, doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      code = hit.at("h3/a").text.strip.sub(/\u25b9/, "").gsub(" / ", "/")
+      c2idx = %r{(?:\(|/)(?<c2>(?:ISO|IEC)\s[^()]+)} =~ code
+      code = code[0...c2idx].strip if c2idx
+      /^(?<code1>[^(]+)(?:\((?<code2>\w+\d+,(?:\sPages)?[^)]+))?/ =~ code
+      if code1.match?(/^CIE/)
+        c = code1.size > 25 && code2 ? "CIE #{code2.sub(/,(\sPages)?/, '')}" : code1
+        add = doc.at("//hgroup/h2")&.text&.match(/(Add)endum\s(\d+)$/)
+        c += " #{add[1]} #{add[2]}" if add
+      elsif (pcode = doc.at('//dt[.="Product Code(s):"]/following-sibling::dd'))
+        c = "CIE #{pcode.text.strip.match(/[^,]+/)}"
+      else
+        num = code.match(/(?<=\()\w{2}\d+,.+(?=\))/).to_s.gsub(/,(?=\s)/, "")
+          .gsub(/,(?=\S)/, " ")
+        c = "CIE #{num}"
+      end
+      docid = [RelatonBib::DocumentIdentifier.new(type: "CIE", id: c)]
+      isbn = doc.at('//dt[contains(.,"ISBN")]/following-sibling::dd')
+      docid << RelatonBib::DocumentIdentifier.new(type: c2.match(/\w+/).to_s, id: c2.strip) if c2
+      docid << RelatonBib::DocumentIdentifier.new(type: "ISBN", id: isbn.text.strip) if isbn
+      docid
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [RelatonBib::TypedTitleStringCollection, Array]
+    def fetch_title(doc)
+      t = doc.at("//hgroup/h2", "//hgroup/h1")
+      return [] unless t
+
+      RelatonBib::TypedTitleString.from_string t.text.strip
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Array<RelatonBib::BibliographicDate>]
+    def fetch_date(doc)
+      doc.xpath('//dt[.="Published:"]/following-sibling::dd[1]').map do |d|
+        pd = d.text.strip
+        on = pd.match?(/^\d{4}(?:[^-]|$)/) ? pd : Date.strptime(pd, "%m/%d/%Y").strftime("%Y-%m-%d")
+        RelatonBib::BibliographicDate.new(type: "published", on: on)
+      end
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [String]
+    def fetch_edition(doc)
+      doc.at('//dt[.="Edition:"]/following-sibling::dd')&.text&.match(/^\d+(?=th)/)&.to_s
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Array<Hash>]
+    def fetch_relation(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      doc.xpath('//section[@class="history"]/ol/li[not(contains(@class,"selected-product"))]').map do |rel|
+        ref = rel.at("a")
+        url = "https://www.techstreet.com#{ref[:href]}"
+        title = RelatonBib::TypedTitleString.from_string ref.at('p/span[@class="title"]').text
+        did = ref.at("h3").text
+        docid = [RelatonBib::DocumentIdentifier.new(type: "CIE", id: did)]
+        on = ref.at("p/time")
+        date = [RelatonBib::BibliographicDate.new(type: "published", on: on[:datetime])]
+        link = [RelatonBib::TypedUri.new(type: "src", content: url)]
+        bibitem = RelatonBib::BibliographicItem.new docid: docid, title: title, link: link, date: date
+        type = ref.at('//li/i[contains(@class,"historical")]') ? "updates" : "updatedBy"
+        { type: type, bibitem: bibitem }
+      end
+    end
+
+    # @param url [String]
+    # @return [Array<RelatonBib::TypedUri>]
+    def fetch_link(url)
+      [RelatonBib::TypedUri.new(type: "src", content: url)]
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Array<RelatonBib::FormattedString>]
+    def fetch_abstract(doc)
+      content = doc.at('//div[contains(@class,"description")]')&.text&.strip
+      return [] if content.nil? || content.empty?
+
+      [RelatonBib::FormattedString.new(content: content, language: "en",
+                                       script: "Latn")]
+    end
+
+    # @param doc [Mechanize::Page]
+    # @return [Array<Hash>]
+    def fetch_contributor(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity
+      authors = doc.xpath('//hgroup/p[not(@class="pub_date")]').text
+      contribs = []
+      until authors.empty?
+        /^(?<sname1>\S+(?:\sder?\s)?[^\s,]+)
+        (?:,?\s(?<sname2>[\w-]{2,})(?=,\s+\w\.))?
+        (?:,?\s(?<fname>[\w-]{2,})(?!,\s+\w\.))?
+        (?:(?:\s?,\s?|\s)(?<init>(?:\w(?:\s?\.|\s|,|$)[\s-]?)+))?
+        (?:(?:,\s?|\s|\.|(?<=\s))(?:and\s)?)?/x =~ authors
+        raise StandardError, "Author name not found in \"#{authors}\"" unless $LAST_MATCH_INFO
+
+        authors.sub! $LAST_MATCH_INFO.to_s, ""
+        sname = [sname1, sname2].compact.join " "
+        surname = RelatonBib::LocalizedString.new sname, "en", "Latn"
+        initial = (init&.strip || "").split(/(?:,|\.)(?:-|\s)?/).map do |int|
+          RelatonBib::LocalizedString.new(int.strip, "en", "Latn")
+        end
+        forename = fname ? [RelatonBib::LocalizedString.new(fname, "en", "Latn")] : []
+        fullname = RelatonBib::FullName.new surname: surname, forename: forename, initial: initial
+        person = RelatonBib::Person.new name: fullname
+        contribs << { entity: person, role: [{ type: "author" }] }
+      end
+      org = RelatonBib::Organization.new(
+        name: "Commission Internationale de L'Eclairage", abbreviation: "CIE",
+        url: "cie.co.at"
+      )
+      contribs << { entity: org, role: [{ type: "publisher" }] }
+    end
+
+    # @param bib [RelatonItu::ItuBibliographicItem]
+    def write_file(bib)
+      id = bib.docidentifier[0].id.gsub(%r{[/\s\-:.]}, "_")
+      file = "#{@output}/#{id.upcase}.#{@format}"
+      # if File.exist? file
+      #   warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
+      #   warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
+      # else
+      out = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
+      File.write file, out, encoding: "UTF-8"
+      # end
+    end
+
+    # @param hit [Nokogiri::HTML::Element]
+    def parse_page(hit) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      url = "https://www.techstreet.com#{hit.at('h3/a')[:href]}"
+      doc = time_req { @agent.get url }
+      item = RelatonBib::BibliographicItem.new(
+        type: "standard", docid: fetch_docid(hit, doc), title: fetch_title(doc),
+        link: fetch_link(url), abstract: fetch_abstract(doc),
+        date: fetch_date(doc), edition: fetch_edition(doc),
+        contributor: fetch_contributor(doc), relation: fetch_relation(doc),
+        language: ["en"], script: ["Latn"], doctype: "document"
+      )
+      write_file item
+    rescue StandardError => e
+      warn "Document: #{url}"
+      warn e.message
+      warn e.backtrace
+    end
+
+    def fetch(url)
+      result = time_req { @agent.get url }
+      result.xpath("//li[@data-product]").each { |hit| parse_page hit }
+      np = result.at '//a[@class="next_page"]'
+      fetch "https://www.techstreet.com#{np[:href]}" if np
+    end
+
+    def time_req
+      t1 = Time.now
+      result = yield
+      t = 1 - (Time.now - t1)
+      sleep t if t.positive?
+      result
+    end
+
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+
+      FileUtils.mkdir output unless Dir.exist? output
+      new(output, format).fetch URL
+
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+  end
+end

data/lib/relaton_cie/processor.rb

@@ -2,11 +2,12 @@ require "relaton/processor"
 
 module RelatonCie
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_cie
       @prefix = "CIE"
       @defaultprefix = /^CIE(-|\s)/
       @idtype = "CIE"
+      @datasets = %w[cie-techstreet]
     end
 
     # @param code [String]
@@ -17,6 +18,18 @@ module RelatonCie
       ::RelatonCie::CieBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the docukents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonBib::BibliographicItem]
     def from_xml(xml)
@@ -26,8 +39,7 @@ module RelatonCie
     # @param hash [Hash]
     # @return [RelatonBib::BibliographicItem]
     def hash_to_bib(hash)
-      item_hash = ::RelatonBib::HashConverter.hash_to_bib(hash)
-      ::RelatonBib::BibliographicItem.new item_hash
+      ::RelatonBib::BibliographicItem.from_hash hash
     end
 
     # Returns hash of XML grammar

data/lib/relaton_cie/scrapper.rb

@@ -6,7 +6,7 @@ module RelatonCie
       # @param code [String]
       # @return [RelatonBib::BibliographicItem]
       def scrape_page(code)
-        url = "#{ENDPOINT}#{code.gsub(/[\/\s\-:\.]/, '_').upcase}.yaml"
+        url = "#{ENDPOINT}#{code.gsub(/[\/\s\-:.]/, '_').upcase}.yaml"
         parse_page url
       rescue OpenURI::HTTPError => e
         return if e.io.status.first == "404"
@@ -21,7 +21,7 @@ module RelatonCie
       def parse_page(url)
         doc = OpenURI.open_uri url
         bib_hash = RelatonBib::HashConverter.hash_to_bib YAML.safe_load(doc)
-        RelatonBib::BibliographicItem.new **bib_hash
+        RelatonBib::BibliographicItem.new(**bib_hash)
       end
     end
   end

data/lib/relaton_cie/version.rb

@@ -1,3 +1,3 @@
 module RelatonCie
-  VERSION = "1.7.pre1".freeze
+  VERSION = "1.9.1".freeze
 end

data/lib/relaton_cie.rb

@@ -4,6 +4,7 @@ require "relaton_bib"
 require "relaton_cie/version"
 require "relaton_cie/cie_bibliography"
 require "relaton_cie/scrapper"
+require "relaton_cie/data_fetcher"
 
 module RelatonCie
   # Returns hash of XML reammar

data/relaton_cie.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   DESCRIPTION
   spec.homepage      = "https://github.com/metanorma/relaton-cie"
   spec.license       = "BSD-2-Clause"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
 
@@ -29,15 +29,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  # spec.add_development_dependency "debase"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
   spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "rake", "~> 10.0"
-  # spec.add_development_dependency "ruby-debug-ide"
   spec.add_development_dependency "ruby-jing"
   spec.add_development_dependency "simplecov"
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
 
-  spec.add_dependency "relaton-bib", "~> 1.7.0"
+  spec.add_dependency "mechanize", "~> 2.8.0"
+  spec.add_dependency "relaton-bib", "~> 1.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-cie
 version: !ruby/object:Gem::Version
-  version: 1.7.pre1
+  version: 1.9.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-03-17 00:00:00.000000000 Z
+date: 2021-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -38,20 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
 - !ruby/object:Gem::Dependency
   name: ruby-jing
   requirement: !ruby/object:Gem::Requirement
@@ -108,20 +94,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: 1.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: 1.9.0
 description: "RelatonEcma: retrieve CIE Standards for bibliographic use \nusing the
   BibliographicItem model.\n"
 email:
@@ -139,6 +139,7 @@ files:
 - README.adoc
 - Rakefile
 - bin/console
+- bin/rspec
 - bin/setup
 - grammars/basicdoc.rng
 - grammars/biblio.rng
@@ -146,6 +147,7 @@ files:
 - grammars/reqt.rng
 - lib/relaton_cie.rb
 - lib/relaton_cie/cie_bibliography.rb
+- lib/relaton_cie/data_fetcher.rb
 - lib/relaton_cie/processor.rb
 - lib/relaton_cie/scrapper.rb
 - lib/relaton_cie/version.rb
@@ -163,14 +165,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: 'RelatonEcma: retrieve CIE Standards for bibliographic use  using the BibliographicItem