relaton-w3c 1.8.0 → 1.9.3

data/grammars/reqt.rng

@@ -0,0 +1,194 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <!--
+    Presupposes isodoc.rnc, is included in it
+    include "isodoc.rnc" { }
+  -->
+  <define name="requirement">
+    <element name="requirement">
+      <ref name="RequirementType"/>
+    </element>
+  </define>
+  <define name="recommendation">
+    <element name="recommendation">
+      <ref name="RequirementType"/>
+    </element>
+  </define>
+  <define name="permission">
+    <element name="permission">
+      <ref name="RequirementType"/>
+    </element>
+  </define>
+  <define name="RequirementType">
+    <optional>
+      <attribute name="obligation">
+        <ref name="ObligationType"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="unnumbered">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="number"/>
+    </optional>
+    <optional>
+      <attribute name="subsequence"/>
+    </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <attribute name="id">
+      <data type="ID"/>
+    </attribute>
+    <optional>
+      <attribute name="filename"/>
+    </optional>
+    <optional>
+      <attribute name="model"/>
+    </optional>
+    <optional>
+      <attribute name="type"/>
+    </optional>
+    <optional>
+      <ref name="reqtitle"/>
+    </optional>
+    <optional>
+      <ref name="label"/>
+    </optional>
+    <optional>
+      <ref name="subject"/>
+    </optional>
+    <zeroOrMore>
+      <ref name="reqinherit"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="classification"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <choice>
+        <ref name="measurementtarget"/>
+        <ref name="specification"/>
+        <ref name="verification"/>
+        <ref name="import"/>
+        <ref name="description"/>
+      </choice>
+    </zeroOrMore>
+    <optional>
+      <ref name="reqt_references"/>
+    </optional>
+    <zeroOrMore>
+      <choice>
+        <ref name="requirement"/>
+        <ref name="recommendation"/>
+        <ref name="permission"/>
+      </choice>
+    </zeroOrMore>
+  </define>
+  <define name="reqtitle">
+    <element name="title">
+      <ref name="FormattedString"/>
+    </element>
+  </define>
+  <define name="label">
+    <element name="label">
+      <text/>
+    </element>
+  </define>
+  <define name="subject">
+    <element name="subject">
+      <text/>
+    </element>
+  </define>
+  <define name="reqinherit">
+    <element name="inherit">
+      <text/>
+    </element>
+  </define>
+  <define name="measurementtarget">
+    <element name="measurement-target">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="specification">
+    <element name="specification">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="verification">
+    <element name="verification">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="import">
+    <element name="import">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="description">
+    <element name="description">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="reqt_references">
+    <element name="references">
+      <oneOrMore>
+        <ref name="bibitem"/>
+      </oneOrMore>
+    </element>
+  </define>
+  <define name="RequirementSubpart">
+    <optional>
+      <attribute name="type"/>
+    </optional>
+    <optional>
+      <attribute name="exclude">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <oneOrMore>
+      <ref name="BasicBlock"/>
+    </oneOrMore>
+  </define>
+  <define name="ObligationType">
+    <choice>
+      <value>requirement</value>
+      <value>recommendation</value>
+      <value>permission</value>
+    </choice>
+  </define>
+  <define name="classification">
+    <element name="classification">
+      <ref name="classification_tag"/>
+      <ref name="classification_value"/>
+    </element>
+  </define>
+  <define name="classification_tag">
+    <element name="tag">
+      <text/>
+    </element>
+  </define>
+  <define name="classification_value">
+    <element name="value">
+      <text/>
+    </element>
+  </define>
+</grammar>

data/lib/relaton_w3c/data_fethcer.rb

@@ -0,0 +1,106 @@
+require "rdf"
+require "linkeddata"
+require "sparql"
+require "mechanize"
+require "relaton_w3c/data_parser"
+
+module RelatonW3c
+  class DataFetcher
+    USED_TYPES = %w[WD NOTE PER PR REC CR].freeze
+
+    attr_reader :data, :group_names
+
+    #
+    # Data fetcher initializer
+    #
+    # @param [String] output directory to save files
+    # @param [String] format format of output files (xml, yaml, bibxml)
+    #
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+      dir = File.dirname(File.expand_path(__FILE__))
+      @group_names = YAML.load_file(File.join(dir , "workgroups.yaml"))
+      @data = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
+      @files = []
+    end
+
+    #
+    # Initialize fetcher and run fetch
+    #
+    # @param [Strin] output directory to save files, default: "data"
+    # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
+    #
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Parse documents
+    #
+    def fetch
+      query.each { |sl| save_doc DataParser.parse(sl, self) }
+    end
+
+    #
+    # Query RDF source for documents
+    #
+    # @return [RDF::Query::Solutions] query results
+    #
+    def query # rubocop:disable Metrics/MethodLength
+      sse = SPARQL.parse(%(
+        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+        PREFIX dc: <http://purl.org/dc/elements/1.1/>
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        # PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
+        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+        SELECT ?link ?title ?date
+        WHERE {
+          ?link dc:title ?title ; dc:date ?date . # ; doc:versionOf ?version_of .
+        }
+      ))
+      data.query sse
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonW3c::W3cBibliographicItem, nil] bib bibliographic item
+    #
+    def save_doc(bib) # rubocop:disable Metrics/MethodLength
+      return unless bib
+
+      c = case @format
+          when "xml" then bib.to_xml(bibdata: true)
+          when "yaml" then bib.to_hash.to_yaml
+          else bib.send("to_#{@format}")
+          end
+      file = file_name(bib)
+      if @files.include? file
+        warn "File #{file} already exists. Document: #{bib.docnumber}"
+      else
+        @files << file
+      end
+      File.write file, c, encoding: "UTF-8"
+    end
+
+    #
+    # Generate file name
+    #
+    # @param [RelatonW3c::W3cBibliographicItem] bib bibliographic item
+    #
+    # @return [String] file name
+    #
+    def file_name(bib)
+      name = bib.docnumber.gsub(/[\s,:\/]/, "_").squeeze("_").upcase
+      File.join @output, "#{name}.#{@ext}"
+    end
+  end
+end

data/lib/relaton_w3c/data_parser.rb

@@ -0,0 +1,205 @@
+module RelatonW3c
+  class DataParser
+    #
+    # Document parser initalization
+    #
+    # @param [RDF::Query::Solution] sol entry from the SPARQL query
+    # @param [RelatonW3c::DataFetcher] fetcher data fetcher
+    #
+    def initialize(sol, fetcher)
+      @sol = sol
+      @fetcher = fetcher
+    end
+
+    #
+    # Initialize document parser and run it
+    #
+    # @param [RDF::Query::Solution] sol entry from the SPARQL query
+    # @param [RelatonW3c::DataFetcher] fetcher data fetcher
+    #
+    # @return [RelatonW3c:W3cBibliographicItem, nil] bibliographic item
+    #
+    def self.parse(sol, fetcher)
+      new(sol, fetcher).parse
+    end
+
+    #
+    # Parse document
+    #
+    # @return [RelatonW3c:W3cBibliographicItem, nil] bibliographic item
+    #
+    def parse # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      return unless @fetcher.class::USED_TYPES.include? type
+
+      RelatonW3c::W3cBibliographicItem.new(
+        type: "standard",
+        doctype: parse_doctype,
+        fetched: Date.today.to_s,
+        language: ["en"],
+        script: ["Latn"],
+        title: parse_title,
+        link: parse_link,
+        docid: parse_docid,
+        docnumber: identifier(@sol.link.to_s),
+        series: parse_series,
+        date: parse_date,
+        relation: parse_relation,
+        contributor: parse_contrib,
+        editorialgroup: parse_editorialgroup,
+      )
+    end
+
+    #
+    # Parse title
+    #
+    # @return [RelatonBib::TypedTitleStringCollection] title
+    #
+    def parse_title
+      t = RelatonBib::TypedTitleString.new content: @sol.title.to_s
+      RelatonBib::TypedTitleStringCollection.new [t]
+    end
+
+    #
+    # Parse link
+    #
+    # @return [Array<RelatonBib::TypedUri>] link
+    #
+    def parse_link
+      [RelatonBib::TypedUri.new(type: "src", content: @sol.link.to_s)]
+    end
+
+    #
+    # Parse docidentifier
+    #
+    # @return [Arra<RelatonBib::DocumentIdentifier>] docidentifier
+    #
+    def parse_docid
+      id = pub_id(@sol.link.to_s)
+      [RelatonBib::DocumentIdentifier.new(type: "W3C", id: id)]
+    end
+
+    #
+    # Generate PubID
+    #
+    # @param [String] url url
+    #
+    # @return [String] PubID
+    #
+    def pub_id(url)
+      "W3C #{identifier(url)}"
+    end
+
+    def identifier(url)
+      /.+\/(\w+(?:-[\w.]+)+(?:\/\w+)?)/.match(url)[1].to_s
+    end
+
+    #
+    # Parse series
+    #
+    # @return [Array<RelatonBib::Series>] series
+    #
+    def parse_series
+      title = RelatonBib::TypedTitleString.new content: "W3C #{type}"
+      [RelatonBib::Series.new(title: title, number: identifier(@sol.link.to_s))]
+    end
+
+    def type # rubocop:disable Metrics/MethodLength
+      @type ||= begin
+        sse = SPARQL.parse(%(
+          PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+          PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+          SELECT ?type
+          WHERE {
+            { <#{@sol.link}> rdf:type ?type }
+          }
+        ))
+        tps = @fetcher.data.query(sse).map { |s| s.type.to_s.split("#").last }
+        tps.detect { |t| Scrapper::DOCTYPES.key?(t) }
+      end
+    end
+
+    #
+    # Parse doctype
+    #
+    # @return [Strinf] doctype
+    #
+    def parse_doctype
+      Scrapper::DOCTYPES[type]
+    end
+
+    def parse_date
+      [RelatonBib::BibliographicDate.new(type: "published", on: @sol.date.to_s)]
+    end
+
+    #
+    # Parse relation
+    #
+    # @return [Array<RelatonBib::DocumentRelation>] relation
+    #
+    def parse_relation # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      sse = SPARQL.parse(%(
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        SELECT ?obsoletes
+        WHERE {
+          VALUES ?p { doc:obsoletes }
+          { <#{@sol.link}> ?p ?obsoletes }
+        }
+      ))
+      @fetcher.data.query(sse).order_by(:obsoletes).map do |r|
+        tp, url = r.to_h.first
+        fr = RelatonBib::LocalizedString.new pub_id(url.to_s)
+        bib = W3cBibliographicItem.new formattedref: fr
+        RelatonBib::DocumentRelation.new(type: tp.to_s, bibitem: bib)
+      end
+    end
+
+    #
+    # Parse contributor
+    #
+    # @return [Array<RelatonBib::ContributionInfo>] contributor
+    #
+    def parse_contrib # rubocop:disable Metrics/MethodLength
+      sse = SPARQL.parse(%(
+        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+        PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>
+        SELECT ?full_name
+        WHERE {
+          <#{@sol.link}> :editor/contact:fullName ?full_name
+        }
+      ))
+      @fetcher.data.query(sse).order_by(:full_name).map do |ed|
+        cn = RelatonBib::LocalizedString.new(ed.full_name.to_s, "en", "Latn")
+        n = RelatonBib::FullName.new completename: cn
+        p = RelatonBib::Person.new name: n
+        RelatonBib::ContributionInfo.new entity: p, role: [type: "editor"]
+      end
+    end
+
+    #
+    # Parse editorialgroup
+    #
+    # @return [RelatonBib::EditorialGroup] editorialgroup
+    #
+    def parse_editorialgroup # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      sse = SPARQL.parse(%(
+        PREFIX org: <http://www.w3.org/2001/04/roadmap/org#>
+        PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>
+        SELECT ?home_page
+        WHERE {
+          <#{@sol.link}> org:deliveredBy/contact:homePage ?home_page
+        }
+      ))
+      res = @fetcher.data.query(sse).order_by(:home_page)
+      tc = res.each_with_object([]) do |edg, obj|
+        wg = @fetcher.group_names[edg.home_page.to_s.sub(/\/$/, "")]
+        if wg
+          rwg = RelatonBib::WorkGroup.new name: wg["name"]
+          obj << RelatonBib::TechnicalCommittee.new(rwg)
+        else
+          warn "Working group name not found for #{edg.home_page}"
+        end
+      end
+      RelatonBib::EditorialGroup.new tc
+    end
+  end
+end

data/lib/relaton_w3c/hash_converter.rb

@@ -4,7 +4,7 @@ module RelatonW3c
       # @param item_hash [Hash]
       # @return [RelatonW3c::W3cBibliographicItem]
       def bib_item(item_hash)
-        W3cBibliographicItem.new **item_hash
+        W3cBibliographicItem.new(**item_hash)
       end
     end
   end

data/lib/relaton_w3c/hit_collection.rb

@@ -22,8 +22,8 @@ module RelatonW3c
     # @param ref [String] reference to search
     def initialize(ref)
       %r{
-        ^(W3C\s)?
-        (?<type>(CR|NOTE|PER|PR|REC|RET|WD|Candidate\sRecommendation|
+        ^(?:W3C\s)?
+        (?<type>(?:CR|NOTE|PER|PR|REC|RET|WD|Candidate\sRecommendation|
           Group\sNote|Proposed\sEdited\sRecommendation|Proposed\sRecommendation|
           Recommendation|Retired|Working\sDraft))? # type
         \s?
@@ -41,7 +41,7 @@ module RelatonW3c
     # @param title_date [String]
     # @param type [String]
     # @return [Array<Hash>]
-    def from_yaml(title_date, type)
+    def from_yaml(title_date, type) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
       /(?<title>.+)\s(?<date>\d{4}-\d{2}-\d{2})$/ =~ title_date
       title ||= title_date
       result = data.select do |hit|
@@ -50,7 +50,7 @@ module RelatonW3c
           type_date_filter(hit, type, date)
       end
       if result.empty?
-        result = data.select { |h| h["link"].split("/").last.match? /#{title}/ }
+        result = data.select { |h| h["link"].split("/").last.match?(/#{title}/) }
       end
       result.map { |h| Hit.new(h, self) }
     end
@@ -60,7 +60,7 @@ module RelatonW3c
     # @param date [String]
     # @return [TrueClass, FalseClass]
     def type_date_filter(hit, type, date) # rubocop:disable Metrics/AbcSize
-      if type && hit["type"] != short_type(type) || date && hit["date"] != date
+      if (type && hit["type"] != short_type(type)) || (date && hit["date"] != date)
         history = get_history hit, type, date
         return false unless history.any?
 
@@ -109,7 +109,7 @@ module RelatonW3c
     # @param type [String]
     # @return [String]
     def short_type(type)
-      tp = TYPES.select { |k,v| v == type }.keys
+      tp = TYPES.select { |_, v| v == type }.keys
       tp.first || type
     end
 
@@ -137,7 +137,7 @@ module RelatonW3c
     # fetch data form server and save it to file.
     #
     def fetch_data
-      resp = Net::HTTP.get_response URI.parse(DOMAIN + "/TR/")
+      resp = Net::HTTP.get_response URI.parse("#{DOMAIN}/TR/")
       # return if there aren't any changes since last fetching
       return unless resp.code == "200"
 
@@ -153,7 +153,7 @@ module RelatonW3c
     # @param h_el [Nokogiri::XML::Element]
     # @param link [Nokogiri::XML::Element]
     # @param pubdetails [Nokogiri::XML::Element]
-    def fetch_hit(h_el, link, pubdetails)
+    def fetch_hit(h_el, link, pubdetails) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       datepub = pubdetails.at("text()").text.match(/\d{4}-\d{2}-\d{2}/).to_s
       editor = h_el.xpath("ul[@class='editorlist']/li").map { |e| e.text.strip }
       keyword = h_el.xpath("ul[@class='taglist']/li").map { |e| e.text.strip }

data/lib/relaton_w3c/processor.rb

@@ -4,11 +4,12 @@ module RelatonW3c
   class Processor < Relaton::Processor
     attr_reader :idtype
 
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_w3c
       @prefix = "W3C"
       @defaultprefix = %r{^W3C\s}
       @idtype = "W3C"
+      @datasets = %w[w3c-rdf]
     end
 
     # @param code [String]
@@ -19,6 +20,18 @@ module RelatonW3c
       ::RelatonW3c::W3cBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from http://www.w3.org/2002/01/tr-automation/tr.rdf
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonCalconnect::CcBibliographicItem]
     def from_xml(xml)
@@ -29,7 +42,7 @@ module RelatonW3c
     # @return [RelatonIsoBib::CcBibliographicItem]
     def hash_to_bib(hash)
       item_hash = ::RelatonW3c::HashConverter.hash_to_bib(hash)
-      ::RelatonW3c::W3cBibliographicItem.new **item_hash
+      ::RelatonW3c::W3cBibliographicItem.new(**item_hash)
     end
 
     # Returns hash of XML grammar