relaton-w3c 1.11.2 → 1.11.5

data/lib/relaton_w3c/w3c_bibliography.rb

@@ -9,10 +9,10 @@ module RelatonW3c
 
     class << self
       # @param text [String]
-      # @return [RelatonW3c::HitCollection]
-      def search(text) # rubocop:disable Metrics/MethodLength
+      # @return [RelatonW3c::W3cBibliographicItem]
+      def search(text) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
         ref = DataParser.parse_identifier text.sub(/^W3C\s/, "")
-        file = DataIndex.create_from_repo.search(ref)
+        file = DataIndex.create_from_repo.search ref.gsub(" ", "-").squeeze("-")
         return unless file
 
         url = "#{SOURCE}#{file}"

data/lib/relaton_w3c.rb

@@ -2,9 +2,6 @@ require "relaton_bib"
 require "relaton_w3c/version"
 require "relaton_w3c/w3c_bibliography"
 require "relaton_w3c/w3c_bibliographic_item"
-# require "relaton_w3c/hit_collection"
-# require "relaton_w3c/hit"
-# require "relaton_w3c/scrapper"
 require "relaton_w3c/xml_parser"
 require "relaton_w3c/bibxml_parser"
 require "relaton_w3c/hash_converter"

data/relaton_w3c.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
                        "using the IsoBibliographicItem model"
   spec.homepage      = "https://github.com/relaton/relaton-wc3"
   spec.license       = "BSD-2-Clause"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.6.0")
 
   # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
 
@@ -39,10 +39,10 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "linkeddata", "~> 3.1.0"
   spec.add_dependency "mechanize", "~> 2.8.0"
-  # spec.add_dependency "picky"
-  spec.add_dependency "rdf", "~> 3.1.0"
-  spec.add_dependency "rdf-normalize", "~> 0.4.0"
+  spec.add_dependency "rdf", "~> 3.2.0"
+  spec.add_dependency "rdf-normalize", "~> 0.5.0"
   spec.add_dependency "relaton-bib", "~> 1.11.0"
+  spec.add_dependency "rubyzip", "~> 2.3.0"
   spec.add_dependency "shex", "~> 0.6.0"
   spec.add_dependency "sparql", "~> 3.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-w3c
 version: !ruby/object:Gem::Version
-  version: 1.11.2
+  version: 1.11.5
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-04-20 00:00:00.000000000 Z
+date: 2022-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -114,28 +114,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: 3.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: 3.2.0
 - !ruby/object:Gem::Dependency
   name: rdf-normalize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +150,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.11.0
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.0
 - !ruby/object:Gem::Dependency
   name: shex
   requirement: !ruby/object:Gem::Requirement
@@ -233,10 +247,7 @@ files:
 - lib/relaton_w3c/data_index.rb
 - lib/relaton_w3c/data_parser.rb
 - lib/relaton_w3c/hash_converter.rb
-- lib/relaton_w3c/hit.rb
-- lib/relaton_w3c/hit_collection.rb
 - lib/relaton_w3c/processor.rb
-- lib/relaton_w3c/scrapper.rb
 - lib/relaton_w3c/version.rb
 - lib/relaton_w3c/w3c_bibliographic_item.rb
 - lib/relaton_w3c/w3c_bibliography.rb
@@ -256,14 +267,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: 'RelatonIso: retrieve W3C Standards for bibliographic using the IsoBibliographicItem

data/lib/relaton_w3c/hit.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module RelatonW3c
-  # Hit.
-  class Hit < RelatonBib::Hit
-    #
-    # Parse page.
-    #
-    # @param lang [String, NilClass]
-    # @return [RelatonW3c::W3cBibliographicItem]
-    def fetch(_lang = nil)
-      @fetch ||= Scrapper.parse_page hit
-    end
-  end
-end

data/lib/relaton_w3c/hit_collection.rb

@@ -1,172 +0,0 @@
-# frozen_string_literal: true
-
-require "fileutils"
-require "yaml"
-
-module RelatonW3c
-  # Page of hit collection.
-  class HitCollection < RelatonBib::HitCollection
-    TYPES = {
-      "CR" => "Candidate Recommendation",
-      "NOTE" => "Group Note",
-      "PER" => "Proposed Edited Recommendation",
-      "PR" => "Proposed Recommendation",
-      "REC" => "Recommendation",
-      "RET" => "Retired",
-      "WD" => "Working Draft",
-    }.freeze
-    DOMAIN = "https://www.w3.org"
-    DATADIR = File.expand_path(".relaton/w3c", Dir.home).freeze
-    DATAFILE = File.expand_path("bibliography.yml", DATADIR).freeze
-
-    # @param ref [String] reference to search
-    def initialize(ref)
-      %r{
-        ^(?:W3C\s)?
-        (?<type>(?:CR|NOTE|PER|PR|REC|RET|WD|Candidate\sRecommendation|
-          Group\sNote|Proposed\sEdited\sRecommendation|Proposed\sRecommendation|
-          Recommendation|Retired|Working\sDraft))? # type
-        \s?
-        (?<title_date>.+) # title_date
-      }x =~ ref
-      super
-      @array = from_yaml title_date, type
-    end
-
-    private
-
-    #
-    # Fetch data form yaml
-    #
-    # @param title_date [String]
-    # @param type [String]
-    # @return [Array<Hash>]
-    def from_yaml(title_date, type) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
-      /(?<title>.+)\s(?<date>\d{4}-\d{2}-\d{2})$/ =~ title_date
-      title ||= title_date
-      result = data.select do |hit|
-        (hit["title"].casecmp?(title) ||
-          hit["link"].split("/").last.match?(/-#{title}-/)) &&
-          type_date_filter(hit, type, date)
-      end
-      if result.empty?
-        result = data.select { |h| h["link"].split("/").last.match?(/#{title}/) }
-      end
-      result.map { |h| Hit.new(h, self) }
-    end
-
-    # @param hit [Hash]
-    # @param type [String]
-    # @param date [String]
-    # @return [TrueClass, FalseClass]
-    def type_date_filter(hit, type, date) # rubocop:disable Metrics/AbcSize
-      if (type && hit["type"] != short_type(type)) || (date && hit["date"] != date)
-        history = get_history hit, type, date
-        return false unless history.any?
-
-        hit["type"] = short_type type
-        hit["datepub"] = history.first.at("td").text
-        hit["link"] = history.first.at("a")[:href]
-      end
-      true
-    end
-
-    # @param hit [Hash]
-    # @param type [String]
-    # @param date [String]
-    # @return [Array<Nokogiri::XML::Element>, Nokogiri::HTML::NodeSet]
-    def get_history(hit, type, date)
-      resp = Net::HTTP.get URI.parse(HitCollection::DOMAIN + hit["history"])
-      history_doc = Nokogiri::HTML resp
-      history = history_doc.xpath(
-        "//table//a[contains(.,'#{long_type(type)}')]/../..",
-      )
-      return filter_history_by_date(history, history_doc, type, date) if date
-
-      history
-    end
-
-    # @param history [Nokogiri::XML::NodeSet]
-    # @param history_doc [Nokogiri::HTML::NodeSet]
-    # @param type [String]
-    # @param date [String]
-    # @return [Array<Nokogiri::XML::Element>, Nokogiri::HTML::NodeSet]
-    def filter_history_by_date(history, history_doc, type, date)
-      if type
-        history.select do |h|
-          h.at("td[@class='table_datecol']").text == date
-        end
-      else
-        history_doc.xpath(
-          "//table//td[@class='table_datecol'][.='#{date}']/..",
-        )
-      end
-    end
-
-    #
-    # Convetr long type name to short
-    #
-    # @param type [String]
-    # @return [String]
-    def short_type(type)
-      tp = TYPES.select { |_, v| v == type }.keys
-      tp.first || type
-    end
-
-    #
-    # Convert shot type name to long
-    #
-    # @param [String]
-    # @return [String]
-    def long_type(type)
-      TYPES[type] || type
-    end
-
-    #
-    # Fetches YAML data
-    #
-    # @return [Hash]
-    def data
-      FileUtils.mkdir_p DATADIR
-      ctime = File.ctime DATAFILE if File.exist? DATAFILE
-      fetch_data if !ctime || ctime.to_date < Date.today
-      @data ||= YAML.safe_load File.read(DATAFILE, encoding: "UTF-8")
-    end
-
-    #
-    # fetch data form server and save it to file.
-    #
-    def fetch_data
-      resp = Net::HTTP.get_response URI.parse("#{DOMAIN}/TR/")
-      # return if there aren't any changes since last fetching
-      return unless resp.code == "200"
-
-      doc = Nokogiri::HTML resp.body
-      @data = doc.xpath("//ul[@id='container']/li").map do |h_el|
-        link = h_el.at("h2/a")
-        pubdetails = h_el.at("p[@class='pubdetails']")
-        fetch_hit h_el, link, pubdetails
-      end
-      File.write DATAFILE, @data.to_yaml, encoding: "UTF-8"
-    end
-
-    # @param h_el [Nokogiri::XML::Element]
-    # @param link [Nokogiri::XML::Element]
-    # @param pubdetails [Nokogiri::XML::Element]
-    def fetch_hit(h_el, link, pubdetails) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      datepub = pubdetails.at("text()").text.match(/\d{4}-\d{2}-\d{2}/).to_s
-      editor = h_el.xpath("ul[@class='editorlist']/li").map { |e| e.text.strip }
-      keyword = h_el.xpath("ul[@class='taglist']/li").map { |e| e.text.strip }
-      {
-        "title" => link.text.gsub("\u00a0", " "),
-        "link" => link[:href],
-        "type" => h_el.at("div").text.upcase,
-        "workgroup" => h_el.xpath("p[@class='deliverer']").map(&:text),
-        "datepub" => datepub,
-        "history" => pubdetails.at("a[text()='History']")[:href],
-        "editor" => editor,
-        "keyword" => keyword,
-      }
-    end
-  end
-end

data/lib/relaton_w3c/scrapper.rb

@@ -1,218 +0,0 @@
-module RelatonW3c
-  class Scrapper
-    DOCTYPES = {
-      "CR" => "candidateRecommendation",
-      "NOTE" => "groupNote",
-      "PER" => "proposedEditedRecommendation",
-      "PR" => "proposedRecommendation",
-      "REC" => "recommendation",
-      "RET" => "retired",
-      "WD" => "workingDraft",
-    }.freeze
-
-    class << self
-      # @param hit [Hash]
-      # @return [RelatonW3c::W3cBibliographicItem]
-      def parse_page(hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-        resp = Net::HTTP.get_response URI.parse(hit["link"])
-        doc = resp.code == "200" ? Nokogiri::HTML(resp.body) : nil
-        W3cBibliographicItem.new(
-          type: "standard",
-          docid: fetch_docid(hit),
-          fetched: Date.today.to_s,
-          language: ["en"],
-          script: ["Latn"],
-          title: fetch_title(hit, doc),
-          abstract: fetch_abstract(doc),
-          link: fetch_link(hit),
-          date: fetch_date(hit, doc),
-          doctype: fetch_doctype(hit, doc),
-          contributor: fetch_contributor(hit, doc),
-          relation: fetch_relation(doc),
-          keyword: hit["keyword"],
-        )
-      end
-
-      private
-
-      # @param hit [Hash]
-      # @return [Array<RelatonBib::DocumentIdentifier>]
-      def fetch_docid(hit)
-        id = hit["link"].split("/").last
-        [RelatonBib::DocumentIdentifier.new(id: id, type: "W3C", primary: true)]
-      end
-
-      # @param hit [Hash]
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<RelatonBib::TypedTitleString>]
-      def fetch_title(hit, doc) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-        titles = []
-        if doc
-          title = doc.at("//*[contains(@id, 'title')]")&.text
-          if title && !title.empty?
-            titles << { content: title.gsub(/\n/, " "), type: "main" }
-          end
-          subtitle = doc.at(
-            "//h2[@id='subtitle']|//p[contains(@class, 'subline')]",
-          )&.text
-          titles << { content: subtitle, tipe: "subtitle" } if subtitle
-        end
-        if titles.empty? && hit["title"]
-          titles << { content: hit["title"], type: "main" }
-        end
-        titles.map do |t|
-          title = RelatonBib::FormattedString.new(
-            content: t[:content], language: "en", script: "Latn",
-          )
-          RelatonBib::TypedTitleString.new(type: t[:type], title: title)
-        end
-      end
-
-      # @param doc [Nokogiri::HTML::Document, NilClass]
-      # @return [Array<RelatonBib::FormattedString>]
-      def fetch_abstract(doc)
-        return [] unless doc
-
-        content = doc.at("//h2[.='Abstract']/following-sibling::p",
-                         "//div[@class='abstract']/p").text
-        [RelatonBib::FormattedString.new(content: content, language: "en",
-                                         script: "Latn")]
-      end
-
-      # @param hit [Hash]
-      # @return [Array<RelatonBib::TypedUri>]
-      def fetch_link(hit)
-        [RelatonBib::TypedUri.new(type: "src", content: hit["link"])]
-      end
-
-      # @param hit [Hash]
-      # @param doc [Nokogiri::HTML::Document, NilClass]
-      # @return [Array<RelatonBib::BibliographicDate>]
-      def fetch_date(hit, doc) # rubocop:disable Metrics/CyclomaticComplexity
-        on = hit["datepub"] || doc&.at("//h2/time[@datetime]")&.attr(:datetime)
-        on ||= fetch_date1(doc) || fetch_date2(doc)
-        [RelatonBib::BibliographicDate.new(type: "published", on: on)] if on
-      end
-
-      # @param doc [Nokogiri::HTML::Document, NilClass]
-      # @return [String]
-      def fetch_date1(doc)
-        d = doc&.at("//h2[@property='dc:issued']")&.attr(:content)
-        d&.match(/\d{4}-\d{2}-\d{2}/)&.to_s
-      end
-
-      # @param doc [Nokogiri::HTML::Document, NilClass]
-      # @return [String]
-      def fetch_date2(doc)
-        d = doc&.at("//h2[contains(@id, 'w3c-recommendation')]")
-        return unless d
-
-        Date.parse(d.attr(:id.match(/\d{2}-\w+-\d{4}/).to_s)).to_s
-      end
-
-      # @param hit [Hash]
-      # @param doc [Nokogiri::HTML::Document, NilClass]
-      # @return [String]
-      def fetch_doctype(hit, doc)
-        if hit["type"]
-          DOCTYPES[hit["type"]]
-        elsif doc
-          type = HitCollection::TYPES.detect do |_k, v|
-            doc.at("//h2[contains(., '#{v}')]/time[@datetime]")
-          end
-          DOCTYPES[type&.first]
-        end
-      end
-
-      # @param hit [Hash]
-      # @param doc [Nokogiri::HTML::Document, NilClass]
-      # @return [Array<RelatonBib::ContributionInfo>]
-      def fetch_contributor(hit, doc) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-        if doc
-          editors = find_contribs(doc, "Editors").reduce([]) do |mem, ed|
-            c = parse_contrib ed, "editor"
-            mem << c if c
-            mem
-          end
-          contribs = find_contribs(doc, "Authors").reduce(editors) do |mem, ath|
-            ed = mem.detect { |e| e[:id] && e[:id] == ath["data-editor-id"] }
-            if ed
-              ed[:role] << { type: "author" }
-            else
-              mem << parse_contrib(ath, "author")
-            end
-            mem
-          end
-          contribs.map { |c| contrib_info(**c) }
-        else
-          hit["editor"].map do |ed|
-            contrib_info name: ed, role: [{ type: "editor" }]
-          end
-        end
-      end
-
-      # @param doc [Nokogiri::NTML::Document]
-      # @param type [String]
-      # @return [Array<Nokogiri::XML::Element]
-      def find_contribs(doc, type)
-        doc.xpath("//dt[contains(.,'#{type}')]/following-sibling::dd"\
-                  "[preceding-sibling::dt[1][contains(.,'#{type}')]]")
-      end
-
-      # @param element [Nokogiri::XML::Element]
-      # @param type [String]
-      # @return [Hash]
-      def parse_contrib(element, type) # rubocop:disable Metrics/MethodLength
-        p = element.at("a")
-        return unless p
-
-        contrib = {
-          name: p.text,
-          url: p[:href],
-          role: [{ type: type }],
-          id: element["data-editor-id"],
-        }
-        org = element.at("a[2]")
-        contrib[:org] = { name: org.text, url: org[:href] } if org
-        contrib
-      end
-
-      # @param name [String]
-      # @param url [String, NilClass]
-      # @param role [Array<Hash>]
-      # @parma org [Hash]
-      # @return [RelatonBib::ContributionInfo]
-      def contrib_info(**args)
-        completename = RelatonBib::LocalizedString.new(args[:name])
-        name = RelatonBib::FullName.new completename: completename
-        af = []
-        if args[:org]
-          org = RelatonBib::Organization.new(**args[:org])
-          af << RelatonBib::Affiliation.new(organization: org)
-        end
-        en = RelatonBib::Person.new name: name, url: args[:url], affiliation: af
-        RelatonBib::ContributionInfo.new entity: en, role: args[:role]
-      end
-
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<RelatonBib::DocumentRelation>]
-      def fetch_relation(doc)
-        return [] unless doc && (link = recommendation_link(doc))
-
-        hit = { "link" => link }
-        item = parse_page hit
-        [RelatonBib::DocumentRelation.new(type: "obsoletedBy", bibitem: item)]
-      end
-
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [String, NilClass]
-      def recommendation_link(doc)
-        recom = doc.at("//dt[.='Latest Recommendation:']",
-                       "//dt[.='Previous Recommendation:']")
-        return unless recom
-
-        recom.at("./following-sibling::dd/a")[:href]
-      end
-    end
-  end
-end