relaton-w3c 1.9.0 → 1.9.4

data/grammars/reqt.rng

@@ -0,0 +1,194 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <!--
+    Presupposes isodoc.rnc, is included in it
+    include "isodoc.rnc" { }
+  -->
+  <define name="requirement">
+    <element name="requirement">
+      <ref name="RequirementType"/>
+    </element>
+  </define>
+  <define name="recommendation">
+    <element name="recommendation">
+      <ref name="RequirementType"/>
+    </element>
+  </define>
+  <define name="permission">
+    <element name="permission">
+      <ref name="RequirementType"/>
+    </element>
+  </define>
+  <define name="RequirementType">
+    <optional>
+      <attribute name="obligation">
+        <ref name="ObligationType"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="unnumbered">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="number"/>
+    </optional>
+    <optional>
+      <attribute name="subsequence"/>
+    </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <attribute name="id">
+      <data type="ID"/>
+    </attribute>
+    <optional>
+      <attribute name="filename"/>
+    </optional>
+    <optional>
+      <attribute name="model"/>
+    </optional>
+    <optional>
+      <attribute name="type"/>
+    </optional>
+    <optional>
+      <ref name="reqtitle"/>
+    </optional>
+    <optional>
+      <ref name="label"/>
+    </optional>
+    <optional>
+      <ref name="subject"/>
+    </optional>
+    <zeroOrMore>
+      <ref name="reqinherit"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="classification"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <choice>
+        <ref name="measurementtarget"/>
+        <ref name="specification"/>
+        <ref name="verification"/>
+        <ref name="import"/>
+        <ref name="description"/>
+      </choice>
+    </zeroOrMore>
+    <optional>
+      <ref name="reqt_references"/>
+    </optional>
+    <zeroOrMore>
+      <choice>
+        <ref name="requirement"/>
+        <ref name="recommendation"/>
+        <ref name="permission"/>
+      </choice>
+    </zeroOrMore>
+  </define>
+  <define name="reqtitle">
+    <element name="title">
+      <ref name="FormattedString"/>
+    </element>
+  </define>
+  <define name="label">
+    <element name="label">
+      <text/>
+    </element>
+  </define>
+  <define name="subject">
+    <element name="subject">
+      <text/>
+    </element>
+  </define>
+  <define name="reqinherit">
+    <element name="inherit">
+      <text/>
+    </element>
+  </define>
+  <define name="measurementtarget">
+    <element name="measurement-target">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="specification">
+    <element name="specification">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="verification">
+    <element name="verification">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="import">
+    <element name="import">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="description">
+    <element name="description">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="reqt_references">
+    <element name="references">
+      <oneOrMore>
+        <ref name="bibitem"/>
+      </oneOrMore>
+    </element>
+  </define>
+  <define name="RequirementSubpart">
+    <optional>
+      <attribute name="type"/>
+    </optional>
+    <optional>
+      <attribute name="exclude">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <oneOrMore>
+      <ref name="BasicBlock"/>
+    </oneOrMore>
+  </define>
+  <define name="ObligationType">
+    <choice>
+      <value>requirement</value>
+      <value>recommendation</value>
+      <value>permission</value>
+    </choice>
+  </define>
+  <define name="classification">
+    <element name="classification">
+      <ref name="classification_tag"/>
+      <ref name="classification_value"/>
+    </element>
+  </define>
+  <define name="classification_tag">
+    <element name="tag">
+      <text/>
+    </element>
+  </define>
+  <define name="classification_value">
+    <element name="value">
+      <text/>
+    </element>
+  </define>
+</grammar>

data/lib/relaton_w3c/data_fethcer.rb

@@ -0,0 +1,106 @@
+require "rdf"
+require "linkeddata"
+require "sparql"
+require "mechanize"
+require "relaton_w3c/data_parser"
+
+module RelatonW3c
+  class DataFetcher
+    USED_TYPES = %w[WD NOTE PER PR REC CR].freeze
+
+    attr_reader :data, :group_names
+
+    #
+    # Data fetcher initializer
+    #
+    # @param [String] output directory to save files
+    # @param [String] format format of output files (xml, yaml, bibxml)
+    #
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+      dir = File.dirname(File.expand_path(__FILE__))
+      @group_names = YAML.load_file(File.join(dir , "workgroups.yaml"))
+      @data = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
+      @files = []
+    end
+
+    #
+    # Initialize fetcher and run fetch
+    #
+    # @param [Strin] output directory to save files, default: "data"
+    # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
+    #
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Parse documents
+    #
+    def fetch
+      query.each { |sl| save_doc DataParser.parse(sl, self) }
+    end
+
+    #
+    # Query RDF source for documents
+    #
+    # @return [RDF::Query::Solutions] query results
+    #
+    def query # rubocop:disable Metrics/MethodLength
+      sse = SPARQL.parse(%(
+        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+        PREFIX dc: <http://purl.org/dc/elements/1.1/>
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        # PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
+        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+        SELECT ?link ?title ?date
+        WHERE {
+          ?link dc:title ?title ; dc:date ?date . # ; doc:versionOf ?version_of .
+        }
+      ))
+      data.query sse
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonW3c::W3cBibliographicItem, nil] bib bibliographic item
+    #
+    def save_doc(bib) # rubocop:disable Metrics/MethodLength
+      return unless bib
+
+      c = case @format
+          when "xml" then bib.to_xml(bibdata: true)
+          when "yaml" then bib.to_hash.to_yaml
+          else bib.send("to_#{@format}")
+          end
+      file = file_name(bib)
+      if @files.include? file
+        warn "File #{file} already exists. Document: #{bib.docnumber}"
+      else
+        @files << file
+      end
+      File.write file, c, encoding: "UTF-8"
+    end
+
+    #
+    # Generate file name
+    #
+    # @param [RelatonW3c::W3cBibliographicItem] bib bibliographic item
+    #
+    # @return [String] file name
+    #
+    def file_name(bib)
+      name = bib.docnumber.gsub(/[\s,:\/]/, "_").squeeze("_").upcase
+      File.join @output, "#{name}.#{@ext}"
+    end
+  end
+end

data/lib/relaton_w3c/data_parser.rb

@@ -0,0 +1,205 @@
+module RelatonW3c
+  class DataParser
+    #
+    # Document parser initalization
+    #
+    # @param [RDF::Query::Solution] sol entry from the SPARQL query
+    # @param [RelatonW3c::DataFetcher] fetcher data fetcher
+    #
+    def initialize(sol, fetcher)
+      @sol = sol
+      @fetcher = fetcher
+    end
+
+    #
+    # Initialize document parser and run it
+    #
+    # @param [RDF::Query::Solution] sol entry from the SPARQL query
+    # @param [RelatonW3c::DataFetcher] fetcher data fetcher
+    #
+    # @return [RelatonW3c:W3cBibliographicItem, nil] bibliographic item
+    #
+    def self.parse(sol, fetcher)
+      new(sol, fetcher).parse
+    end
+
+    #
+    # Parse document
+    #
+    # @return [RelatonW3c:W3cBibliographicItem, nil] bibliographic item
+    #
+    def parse # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      return unless @fetcher.class::USED_TYPES.include? type
+
+      RelatonW3c::W3cBibliographicItem.new(
+        type: "standard",
+        doctype: parse_doctype,
+        fetched: Date.today.to_s,
+        language: ["en"],
+        script: ["Latn"],
+        title: parse_title,
+        link: parse_link,
+        docid: parse_docid,
+        docnumber: identifier(@sol.link.to_s),
+        series: parse_series,
+        date: parse_date,
+        relation: parse_relation,
+        contributor: parse_contrib,
+        editorialgroup: parse_editorialgroup,
+      )
+    end
+
+    #
+    # Parse title
+    #
+    # @return [RelatonBib::TypedTitleStringCollection] title
+    #
+    def parse_title
+      t = RelatonBib::TypedTitleString.new content: @sol.title.to_s
+      RelatonBib::TypedTitleStringCollection.new [t]
+    end
+
+    #
+    # Parse link
+    #
+    # @return [Array<RelatonBib::TypedUri>] link
+    #
+    def parse_link
+      [RelatonBib::TypedUri.new(type: "src", content: @sol.link.to_s)]
+    end
+
+    #
+    # Parse docidentifier
+    #
+    # @return [Arra<RelatonBib::DocumentIdentifier>] docidentifier
+    #
+    def parse_docid
+      id = pub_id(@sol.link.to_s)
+      [RelatonBib::DocumentIdentifier.new(type: "W3C", id: id)]
+    end
+
+    #
+    # Generate PubID
+    #
+    # @param [String] url url
+    #
+    # @return [String] PubID
+    #
+    def pub_id(url)
+      "W3C #{identifier(url)}"
+    end
+
+    def identifier(url)
+      /.+\/(\w+(?:-[\w.]+)+(?:\/\w+)?)/.match(url)[1].to_s
+    end
+
+    #
+    # Parse series
+    #
+    # @return [Array<RelatonBib::Series>] series
+    #
+    def parse_series
+      title = RelatonBib::TypedTitleString.new content: "W3C #{type}"
+      [RelatonBib::Series.new(title: title, number: identifier(@sol.link.to_s))]
+    end
+
+    def type # rubocop:disable Metrics/MethodLength
+      @type ||= begin
+        sse = SPARQL.parse(%(
+          PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+          PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+          SELECT ?type
+          WHERE {
+            { <#{@sol.link}> rdf:type ?type }
+          }
+        ))
+        tps = @fetcher.data.query(sse).map { |s| s.type.to_s.split("#").last }
+        tps.detect { |t| Scrapper::DOCTYPES.key?(t) }
+      end
+    end
+
+    #
+    # Parse doctype
+    #
+    # @return [Strinf] doctype
+    #
+    def parse_doctype
+      Scrapper::DOCTYPES[type]
+    end
+
+    def parse_date
+      [RelatonBib::BibliographicDate.new(type: "published", on: @sol.date.to_s)]
+    end
+
+    #
+    # Parse relation
+    #
+    # @return [Array<RelatonBib::DocumentRelation>] relation
+    #
+    def parse_relation # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      sse = SPARQL.parse(%(
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        SELECT ?obsoletes
+        WHERE {
+          VALUES ?p { doc:obsoletes }
+          { <#{@sol.link}> ?p ?obsoletes }
+        }
+      ))
+      @fetcher.data.query(sse).order_by(:obsoletes).map do |r|
+        tp, url = r.to_h.first
+        fr = RelatonBib::LocalizedString.new pub_id(url.to_s)
+        bib = W3cBibliographicItem.new formattedref: fr
+        RelatonBib::DocumentRelation.new(type: tp.to_s, bibitem: bib)
+      end
+    end
+
+    #
+    # Parse contributor
+    #
+    # @return [Array<RelatonBib::ContributionInfo>] contributor
+    #
+    def parse_contrib # rubocop:disable Metrics/MethodLength
+      sse = SPARQL.parse(%(
+        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+        PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>
+        SELECT ?full_name
+        WHERE {
+          <#{@sol.link}> :editor/contact:fullName ?full_name
+        }
+      ))
+      @fetcher.data.query(sse).order_by(:full_name).map do |ed|
+        cn = RelatonBib::LocalizedString.new(ed.full_name.to_s, "en", "Latn")
+        n = RelatonBib::FullName.new completename: cn
+        p = RelatonBib::Person.new name: n
+        RelatonBib::ContributionInfo.new entity: p, role: [type: "editor"]
+      end
+    end
+
+    #
+    # Parse editorialgroup
+    #
+    # @return [RelatonBib::EditorialGroup] editorialgroup
+    #
+    def parse_editorialgroup # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      sse = SPARQL.parse(%(
+        PREFIX org: <http://www.w3.org/2001/04/roadmap/org#>
+        PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>
+        SELECT ?home_page
+        WHERE {
+          <#{@sol.link}> org:deliveredBy/contact:homePage ?home_page
+        }
+      ))
+      res = @fetcher.data.query(sse).order_by(:home_page)
+      tc = res.each_with_object([]) do |edg, obj|
+        wg = @fetcher.group_names[edg.home_page.to_s.sub(/\/$/, "")]
+        if wg
+          rwg = RelatonBib::WorkGroup.new name: wg["name"]
+          obj << RelatonBib::TechnicalCommittee.new(rwg)
+        else
+          warn "Working group name not found for #{edg.home_page}"
+        end
+      end
+      RelatonBib::EditorialGroup.new tc
+    end
+  end
+end

data/lib/relaton_w3c/hit_collection.rb

@@ -60,7 +60,7 @@ module RelatonW3c
     # @param date [String]
     # @return [TrueClass, FalseClass]
     def type_date_filter(hit, type, date) # rubocop:disable Metrics/AbcSize
-      if type && hit["type"] != short_type(type) || date && hit["date"] != date
+      if (type && hit["type"] != short_type(type)) || (date && hit["date"] != date)
         history = get_history hit, type, date
         return false unless history.any?
 

data/lib/relaton_w3c/processor.rb

@@ -4,11 +4,12 @@ module RelatonW3c
   class Processor < Relaton::Processor
     attr_reader :idtype
 
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_w3c
       @prefix = "W3C"
       @defaultprefix = %r{^W3C\s}
       @idtype = "W3C"
+      @datasets = %w[w3c-rdf]
     end
 
     # @param code [String]
@@ -19,6 +20,18 @@ module RelatonW3c
       ::RelatonW3c::W3cBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from http://www.w3.org/2002/01/tr-automation/tr.rdf
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonCalconnect::CcBibliographicItem]
     def from_xml(xml)

data/lib/relaton_w3c/scrapper.rb

@@ -1,16 +1,16 @@
 module RelatonW3c
   class Scrapper
-    class << self
-      DOCTYPES = {
-        "CR" => "candidateRecommendation",
-        "NOTE" => "groupNote",
-        "PER" => "proposedEditedRecommendation",
-        "PR" => "proposedRecommendation",
-        "REC" => "recommendation",
-        "RET" => "retired",
-        "WD" => "workingDraft",
-      }.freeze
+    DOCTYPES = {
+      "CR" => "candidateRecommendation",
+      "NOTE" => "groupNote",
+      "PER" => "proposedEditedRecommendation",
+      "PR" => "proposedRecommendation",
+      "REC" => "recommendation",
+      "RET" => "retired",
+      "WD" => "workingDraft",
+    }.freeze
 
+    class << self
       # @param hit [Hash]
       # @return [RelatonW3c::W3cBibliographicItem]
       def parse_page(hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength

data/lib/relaton_w3c/version.rb

@@ -1,3 +1,3 @@
 module RelatonW3c
-  VERSION = "1.9.0".freeze
+  VERSION = "1.9.4".freeze
 end

data/lib/relaton_w3c/w3c_bibliography.rb

@@ -5,11 +5,21 @@ require "net/http"
 module RelatonW3c
   # Class methods for search W3C standards.
   class W3cBibliography
+    SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-w3c/main/data/"
+
     class << self
       # @param text [String]
       # @return [RelatonW3c::HitCollection]
-      def search(text)
-        HitCollection.new text
+      def search(text) # rubocop:disable Metrics/MethodLength
+        # HitCollection.new text
+        file = text.sub(/^W3C\s/, "").gsub(/[\s,:\/]/, "_").squeeze("_").upcase
+        url = "#{SOURCE}#{file}.yaml"
+        resp = Net::HTTP.get_response(URI.parse(url))
+        return unless resp.code == "200"
+
+        hash = YAML.safe_load resp.body
+        item_hash = ::RelatonW3c::HashConverter.hash_to_bib(hash)
+        ::RelatonW3c::W3cBibliographicItem.new(**item_hash)
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
              EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
              Net::ProtocolError, Errno::ETIMEDOUT
@@ -24,11 +34,14 @@ module RelatonW3c
       def get(ref, _year = nil, _opts = {})
         warn "[relaton-w3c] (\"#{ref}\") fetching..."
         result = search(ref)
-        return unless result.any?
+        unless result
+          warn "[relaton-w3c] (\"#{ref}\") not found."
+          return
+        end
 
-        ret = result.first.fetch
-        warn "[relaton-w3c] (\"#{ref}\") found #{ret.title.first.title.content}"
-        ret
+        # ret = result.first.fetch
+        warn "[relaton-w3c] (\"#{ref}\") found #{result.title.first.title.content}"
+        result
       end
     end
   end