relaton-w3c 1.10.1 → 1.11.2

data/lib/relaton_w3c/data_parser.rb

@@ -1,5 +1,24 @@
 module RelatonW3c
   class DataParser
+    USED_TYPES = %w[WD NOTE PER PR REC CR].freeze
+
+    DOCTYPES = {
+      "TR" => "technicalReport",
+      "NOTE" => "groupNote",
+    }.freeze
+
+    STAGES = {
+      "RET" => "retired",
+      "SPSD" => "supersededRecommendation",
+      "OBSL" => "obsoletedRecommendation",
+      "WD" => "workingDraft",
+      "CRD" => "candidateRecommendationDraft",
+      "CR" => "candidateRecommendation",
+      "PR" => "proposedRecommendation",
+      "PER" => "proposedEditedRecommendation",
+      "REC" => "recommendation",
+    }.freeze
+
     #
     # Document parser initalization
     #
@@ -29,7 +48,7 @@ module RelatonW3c
     # @return [RelatonW3c:W3cBibliographicItem, nil] bibliographic item
     #
     def parse # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      return unless @fetcher.class::USED_TYPES.include? type
+      return if @sol.respond_to?(:link) && !types_stages.detect { |ts| USED_TYPES.include?(ts) }
 
       RelatonW3c::W3cBibliographicItem.new(
         type: "standard",
@@ -37,10 +56,12 @@ module RelatonW3c
         fetched: Date.today.to_s,
         language: ["en"],
         script: ["Latn"],
+        docstatus: parse_docstatus,
         title: parse_title,
         link: parse_link,
         docid: parse_docid,
-        docnumber: identifier(@sol.link.to_s),
+        formattedref: parse_formattedref,
+        docnumber: identifier,
         series: parse_series,
         date: parse_date,
         relation: parse_relation,
@@ -49,12 +70,24 @@ module RelatonW3c
       )
     end
 
+    #
+    # Extract documetn status
+    #
+    # @return [RelatonBib::DocumentStatus, nil] dcoument status
+    #
+    def parse_docstatus
+      stage = types_stages&.detect { |st| STAGES.include?(st) }
+      RelatonBib::DocumentStatus.new stage: STAGES[stage] if stage
+    end
+
     #
     # Parse title
     #
     # @return [RelatonBib::TypedTitleStringCollection] title
     #
     def parse_title
+      return [] unless @sol.respond_to?(:title)
+
       t = RelatonBib::TypedTitleString.new content: @sol.title.to_s
       RelatonBib::TypedTitleStringCollection.new [t]
     end
@@ -65,7 +98,9 @@ module RelatonW3c
     # @return [Array<RelatonBib::TypedUri>] link
     #
     def parse_link
-      [RelatonBib::TypedUri.new(type: "src", content: @sol.link.to_s)]
+      link = @sol.respond_to?(:link) ? @sol.link : @sol.version_of
+
+      [RelatonBib::TypedUri.new(type: "src", content: link.to_s)]
     end
 
     #
@@ -74,23 +109,45 @@ module RelatonW3c
     # @return [Arra<RelatonBib::DocumentIdentifier>] docidentifier
     #
     def parse_docid
-      id = pub_id(@sol.link.to_s)
+      return [] unless @sol.respond_to?(:link)
+
+      id = pub_id(@sol.link)
       [RelatonBib::DocumentIdentifier.new(type: "W3C", id: id, primary: true)]
     end
 
     #
     # Generate PubID
     #
-    # @param [String] url url
-    #
-    # @return [String] PubID
+    # @return [RDF::URI] PubID
     #
     def pub_id(url)
       "W3C #{identifier(url)}"
     end
 
-    def identifier(url)
-      /.+\/(\w+(?:-[\w.]+)+(?:\/\w+)?)/.match(url)[1].to_s
+    #
+    # Generate identifier from URL
+    #
+    # @param [RDF::URI, nil] link
+    #
+    # @return [String] identifier
+    #
+    def identifier(link = nil)
+      url = link || (@sol.respond_to?(:link) ? @sol.link : @sol.version_of)
+      self.class.parse_identifier(url.to_s)
+    end
+
+    #
+    # Parse identifier from URL
+    #
+    # @param [String] url URL
+    #
+    # @return [String] identifier
+    #
+    def self.parse_identifier(url)
+      if /.+\/(\w+(?:-[\w.]+)+(?:\/\w+)?)/ =~ url.to_s
+        $1.to_s
+      else url.to_s.split("/").last
+      end
     end
 
     #
@@ -99,12 +156,31 @@ module RelatonW3c
     # @return [Array<RelatonBib::Series>] series
     #
     def parse_series
+      return [] unless type
+
       title = RelatonBib::TypedTitleString.new content: "W3C #{type}"
-      [RelatonBib::Series.new(title: title, number: identifier(@sol.link.to_s))]
+      [RelatonBib::Series.new(title: title, number: identifier)]
     end
 
-    def type # rubocop:disable Metrics/MethodLength
-      @type ||= begin
+    #
+    # Extract type
+    #
+    # @return [String] type
+    #
+    def type
+      # thre are many types, we need to find the right one
+      @type ||= types_stages&.detect { |t| USED_TYPES.include?(t) }
+    end
+
+    #
+    # Fetches types and stages
+    #
+    # @return [Array<String>] types and stages
+    #
+    def types_stages # rubocop:disable Metrics/MethodLength
+      return unless @sol.respond_to?(:link)
+
+      @types_stages ||= begin
         sse = SPARQL.parse(%(
           PREFIX : <http://www.w3.org/2001/02pd/rec54#>
           PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
@@ -113,8 +189,7 @@ module RelatonW3c
             { <#{@sol.link}> rdf:type ?type }
           }
         ))
-        tps = @fetcher.data.query(sse).map { |s| s.type.to_s.split("#").last }
-        tps.detect { |t| Scrapper::DOCTYPES.key?(t) }
+        @fetcher.data.query(sse).map { |s| s.type.to_s.split("#").last }
       end
     end
 
@@ -124,10 +199,17 @@ module RelatonW3c
     # @return [Strinf] doctype
     #
     def parse_doctype
-      Scrapper::DOCTYPES[type]
+      DOCTYPES[type] || "recommendation"
     end
 
+    #
+    # Parse date
+    #
+    # @return [Array<RelatonBib::BibliographicDate>] date
+    #
     def parse_date
+      return [] unless @sol.respond_to?(:date)
+
       [RelatonBib::BibliographicDate.new(type: "published", on: @sol.date.to_s)]
     end
 
@@ -136,29 +218,90 @@ module RelatonW3c
     #
     # @return [Array<RelatonBib::DocumentRelation>] relation
     #
-    def parse_relation # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+    def parse_relation
+      if @sol.respond_to?(:link)
+        relations + editor_drafts
+      else document_versions
+      end
+    end
+
+    def relations # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      {
+        "doc:obsoletes" => { type: "obsoletes" },
+        "mat:hasErrata" => { type: "updatedBy", description: "errata" },
+        # "mat:hasTranslations" => "hasTranslation",
+        # "mat:hasImplReport" => "hasImpReport",
+        ":previousEdition" => { type: "editionOf" },
+      }.reduce([]) do |acc, (predicate, tp)|
+        acc + relation_query(predicate).map do |r|
+          fr = RelatonBib::LocalizedString.new pub_id(r.rel.to_s)
+          bib = W3cBibliographicItem.new formattedref: fr
+          tp[:description] = RelatonBib::FormattedString.new content: tp[:description] if tp[:description]
+          RelatonBib::DocumentRelation.new(**tp, bibitem: bib)
+        end
+      end
+    end
+
+    def editor_drafts # rubocop:disable Metrics/MethodLength
       sse = SPARQL.parse(%(
-        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
-        SELECT ?obsoletes
-        WHERE {
-          VALUES ?p { doc:obsoletes }
-          { <#{@sol.link}> ?p ?obsoletes }
-        }
+        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+        SELECT ?rel
+        WHERE { <#{@sol.link}> :ED ?rel . }
       ))
-      @fetcher.data.query(sse).order_by(:obsoletes).map do |r|
-        tp, url = r.to_h.first
-        fr = RelatonBib::LocalizedString.new pub_id(url.to_s)
+      @fetcher.data.query(sse).map do |s|
+        fr = RelatonBib::LocalizedString.new pub_id(s.rel.to_s)
         bib = W3cBibliographicItem.new formattedref: fr
-        RelatonBib::DocumentRelation.new(type: tp.to_s, bibitem: bib)
+        desc = RelatonBib::FormattedString.new content: "Editor's draft"
+        RelatonBib::DocumentRelation.new(
+          type: "hasDraft", description: desc, bibitem: bib,
+        )
+      end
+    end
+
+    def relation_query(predicate)
+      sse = SPARQL.parse(%(
+        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
+        SELECT ?rel
+        WHERE { <#{@sol.link}> #{predicate} ?rel . }
+      ))
+      @fetcher.data.query(sse).order_by(:rel)
+    end
+
+    def document_versions
+      sse = SPARQL.parse(%(
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        SELECT ?link
+        WHERE { ?link doc:versionOf <#{@sol.version_of}> }
+      ))
+      @fetcher.data.query(sse).map do |r|
+        fref = RelatonBib::FormattedRef.new content: pub_id(r.link)
+        bib = W3cBibliographicItem.new formattedref: fref
+        RelatonBib::DocumentRelation.new(type: "hasEdition", bibitem: bib)
       end
     end
 
+    #
+    # Parse formattedref
+    #
+    # @return [RelatonBib::FormattedRef] formattedref
+    #
+    def parse_formattedref
+      return if @sol.respond_to?(:link)
+
+      RelatonBib::FormattedRef.new(content: pub_id(@sol.version_of))
+    end
+
     #
     # Parse contributor
     #
     # @return [Array<RelatonBib::ContributionInfo>] contributor
     #
     def parse_contrib # rubocop:disable Metrics/MethodLength
+      return [] unless @sol.respond_to?(:link)
+
       sse = SPARQL.parse(%(
         PREFIX : <http://www.w3.org/2001/02pd/rec54#>
         PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>
@@ -181,6 +324,8 @@ module RelatonW3c
     # @return [RelatonBib::EditorialGroup] editorialgroup
     #
     def parse_editorialgroup # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      return unless @sol.respond_to?(:link)
+
       sse = SPARQL.parse(%(
         PREFIX org: <http://www.w3.org/2001/04/roadmap/org#>
         PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>

data/lib/relaton_w3c/version.rb

@@ -1,3 +1,3 @@
 module RelatonW3c
-  VERSION = "1.10.1".freeze
+  VERSION = "1.11.2".freeze
 end

data/lib/relaton_w3c/w3c_bibliography.rb

@@ -5,15 +5,17 @@ require "net/http"
 module RelatonW3c
   # Class methods for search W3C standards.
   class W3cBibliography
-    SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-w3c/main/data/"
+    SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-w3c/main/"
 
     class << self
       # @param text [String]
       # @return [RelatonW3c::HitCollection]
       def search(text) # rubocop:disable Metrics/MethodLength
-        # HitCollection.new text
-        file = text.sub(/^W3C\s/, "").gsub(/[\s,:\/]/, "_").squeeze("_").upcase
-        url = "#{SOURCE}#{file}.yaml"
+        ref = DataParser.parse_identifier text.sub(/^W3C\s/, "")
+        file = DataIndex.create_from_repo.search(ref)
+        return unless file
+
+        url = "#{SOURCE}#{file}"
         resp = Net::HTTP.get_response(URI.parse(url))
         return unless resp.code == "200"
 
@@ -24,7 +26,7 @@ module RelatonW3c
              EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
              Net::ProtocolError, Errno::ETIMEDOUT
         raise RelatonBib::RequestError,
-              "Could not access #{HitCollection::DOMAIN}"
+              "Could not access #{url}"
       end
 
       # @param ref [String] the W3C standard Code to look up
@@ -39,8 +41,8 @@ module RelatonW3c
           return
         end
 
-        # ret = result.first.fetch
-        warn "[relaton-w3c] (\"#{ref}\") found #{result.title.first.title.content}"
+        found = result.docnumber
+        warn "[relaton-w3c] (\"#{ref}\") found #{found}"
         result
       end
     end

data/lib/relaton_w3c/workgroups.yaml

@@ -32,6 +32,9 @@
 'https://www.w3.org/WAI/EO':
   name: Education and Outreach Working Group 
   abbrev: EOWG
+'https://www.w3.org/WAI/about/groups/eowg':
+  name: Education and Outreach Working Group 
+  abbrev: EOWG
 'https://www.w3.org/2001/sw/WebOnt':
   name: Web-Ontology Working Group
 'http://www.w3.org/MarkUp/Forms':
@@ -54,6 +57,8 @@
   name: Web Applications Working Group
 'https://www.w3.org/2008/webapps':
   name: Web Applications Working Group
+'https://www.w3.org/groups/wg/webapps':
+  name: Web Applications Working Group
 'https://www.w3.org/das':
   name: Devices and Sensors Working Group
   abbrev: DAS WG
@@ -226,6 +231,8 @@
   abbrev: ARIA WG
 'https://www.w3.org/wasm':
   name: WebAssembly Working Group
+'https://www.w3.org/groups/wg/wasm':
+  name: WebAssembly Working Group
 'https://www.w3.org/groups/wg/webediting':
   name: Web Editing Working Group
 'https://www.w3.org/2014/data-shapes':

data/lib/relaton_w3c.rb

@@ -2,13 +2,14 @@ require "relaton_bib"
 require "relaton_w3c/version"
 require "relaton_w3c/w3c_bibliography"
 require "relaton_w3c/w3c_bibliographic_item"
-require "relaton_w3c/hit_collection"
-require "relaton_w3c/hit"
-require "relaton_w3c/scrapper"
+# require "relaton_w3c/hit_collection"
+# require "relaton_w3c/hit"
+# require "relaton_w3c/scrapper"
 require "relaton_w3c/xml_parser"
 require "relaton_w3c/bibxml_parser"
 require "relaton_w3c/hash_converter"
-require "relaton_w3c/data_fethcer"
+require "relaton_w3c/data_fetcher"
+require "relaton_w3c/data_index"
 
 module RelatonW3c
   class Error < StandardError; end

data/relaton_w3c.gemspec

@@ -39,7 +39,10 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "linkeddata", "~> 3.1.0"
   spec.add_dependency "mechanize", "~> 2.8.0"
+  # spec.add_dependency "picky"
   spec.add_dependency "rdf", "~> 3.1.0"
-  spec.add_dependency "relaton-bib", "~> 1.10.1"
+  spec.add_dependency "rdf-normalize", "~> 0.4.0"
+  spec.add_dependency "relaton-bib", "~> 1.11.0"
+  spec.add_dependency "shex", "~> 0.6.0"
   spec.add_dependency "sparql", "~> 3.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-w3c
 version: !ruby/object:Gem::Version
-  version: 1.10.1
+  version: 1.11.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-02-08 00:00:00.000000000 Z
+date: 2022-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -122,20 +122,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: rdf-normalize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.10.1
+        version: 1.11.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+- !ruby/object:Gem::Dependency
+  name: shex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.10.1
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: sparql
   requirement: !ruby/object:Gem::Requirement
@@ -201,7 +229,8 @@ files:
 - grammars/reqt.rng
 - lib/relaton_w3c.rb
 - lib/relaton_w3c/bibxml_parser.rb
-- lib/relaton_w3c/data_fethcer.rb
+- lib/relaton_w3c/data_fetcher.rb
+- lib/relaton_w3c/data_index.rb
 - lib/relaton_w3c/data_parser.rb
 - lib/relaton_w3c/hash_converter.rb
 - lib/relaton_w3c/hit.rb
@@ -234,7 +263,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: 'RelatonIso: retrieve W3C Standards for bibliographic using the IsoBibliographicItem

data/lib/relaton_w3c/data_fethcer.rb

@@ -1,110 +0,0 @@
-require "rdf"
-require "linkeddata"
-require "sparql"
-require "mechanize"
-require "relaton_w3c/data_parser"
-
-module RelatonW3c
-  class DataFetcher
-    USED_TYPES = %w[WD NOTE PER PR REC CR].freeze
-
-    attr_reader :data, :group_names
-
-    #
-    # Data fetcher initializer
-    #
-    # @param [String] output directory to save files
-    # @param [String] format format of output files (xml, yaml, bibxml)
-    #
-    def initialize(output, format)
-      @output = output
-      @format = format
-      @ext = format.sub(/^bib/, "")
-      dir = File.dirname(File.expand_path(__FILE__))
-      @group_names = YAML.load_file(File.join(dir , "workgroups.yaml"))
-      @data = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
-      @files = []
-    end
-
-    #
-    # Initialize fetcher and run fetch
-    #
-    # @param [Strin] output directory to save files, default: "data"
-    # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
-    #
-    def self.fetch(output: "data", format: "yaml")
-      t1 = Time.now
-      puts "Started at: #{t1}"
-      FileUtils.mkdir_p output unless Dir.exist? output
-      new(output, format).fetch
-      t2 = Time.now
-      puts "Stopped at: #{t2}"
-      puts "Done in: #{(t2 - t1).round} sec."
-    end
-
-    #
-    # Parse documents
-    #
-    def fetch
-      query.each { |sl| save_doc DataParser.parse(sl, self) }
-      Dir[File.expand_path("../../data/*", __dir__)].each do |file|
-        xml = File.read file, encoding: "UTF-8"
-        save_doc BibXMLParser.parse(xml)
-      end
-    end
-
-    #
-    # Query RDF source for documents
-    #
-    # @return [RDF::Query::Solutions] query results
-    #
-    def query # rubocop:disable Metrics/MethodLength
-      sse = SPARQL.parse(%(
-        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
-        PREFIX dc: <http://purl.org/dc/elements/1.1/>
-        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
-        # PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
-        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
-        SELECT ?link ?title ?date
-        WHERE {
-          ?link dc:title ?title ; dc:date ?date . # ; doc:versionOf ?version_of .
-        }
-      ))
-      data.query sse
-    end
-
-    #
-    # Save document to file
-    #
-    # @param [RelatonW3c::W3cBibliographicItem, nil] bib bibliographic item
-    #
-    def save_doc(bib) # rubocop:disable Metrics/MethodLength
-      return unless bib
-
-      c = case @format
-          when "xml" then bib.to_xml(bibdata: true)
-          when "yaml" then bib.to_hash.to_yaml
-          else bib.send("to_#{@format}")
-          end
-      file = file_name(bib)
-      if @files.include? file
-        warn "File #{file} already exists. Document: #{bib.docnumber}"
-      else
-        @files << file
-      end
-      File.write file, c, encoding: "UTF-8"
-    end
-
-    #
-    # Generate file name
-    #
-    # @param [RelatonW3c::W3cBibliographicItem] bib bibliographic item
-    #
-    # @return [String] file name
-    #
-    def file_name(bib)
-      name = bib.docnumber.gsub(/[\s,:\/]/, "_").squeeze("_").upcase
-      File.join @output, "#{name}.#{@ext}"
-    end
-  end
-end