relaton-w3c 1.11.0 → 1.11.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ce3bb960545ad6330d96829be3142732ba1c756e24b0e339ca52a094d7c9447
-  data.tar.gz: 8659ebfc94938514ca01a3f202f8b70fe4926268eedf7cc8cb83ef521eda5c8a
+  metadata.gz: 4c96856baa51c84d8397b62be6dee1b8225e7ab854cd70176187e68fc269a23c
+  data.tar.gz: 54d5cc018e0de6ecca6d402ccbe41947047190d08e922c8f46b7cb20164cea22
 SHA512:
-  metadata.gz: 5620a3934fd79f4307489fc8c8129cf8f3c601aa71e3ef0ca568db8ced117bdc2260b392f8711c80f605930fa8fcc5a7f4b0fdf2cd72b551a7747d5bff902ce2
-  data.tar.gz: 71deae107a6f79a4633822855cdf70f4f6e693601f96bbb5f323642cdb78b8f686057793f8483dd012c1ea530adcce8d1d1ab405b8faae30d94a7e1c71def229
+  metadata.gz: 070eb14907a49f99b7c0f45841d83250c1c051cd9a16b9cddfc83f6d874274f401ac95c9554f6994c6702e557ef8a200e576ee5fe1d0b7c7f7583afb53074794
+  data.tar.gz: cab9e48e248b889c5d15e3449a93dc5cb100b7680a87f025ebfd090a1afba36e77730d9e0bc85a10bc9208b9ea2d813e444e0a28b2ac77d412650395479468f3

data/data/reference.W3C.P3P.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding='UTF-8'?>
-<reference anchor="W3C.P3P" taret="http://www.w3.org/TR/P3P/">
+<reference anchor="W3C.P3P" target="http://www.w3.org/TR/P3P/">
   <front>
     <title>The Platform for Privacy Preferences 1.0 (P3P1.0) Specification</title>
     <author initials="M." surname="Marchiori" fullname="Massimo Marchiori">

data/lib/relaton_w3c/bibxml_parser.rb

@@ -13,5 +13,12 @@ module RelatonW3c
     def pubid_type(_)
       "W3C"
     end
+
+    def docids(reference, ver)
+      ids = super
+      ids.reject! &:primary
+      id = "W3C #{reference[:target].split('/').last}"
+      ids.unshift RelatonBib::DocumentIdentifier.new(id: id, type: "W3C", primary: true)
+    end
   end
 end

data/lib/relaton_w3c/data_fetcher.rb

@@ -0,0 +1,188 @@
+require "rdf"
+require "linkeddata"
+require "sparql"
+require "mechanize"
+require "relaton_w3c/data_parser"
+
+module RelatonW3c
+  class DataFetcher
+    attr_reader :data, :group_names
+
+    #
+    # Data fetcher initializer
+    #
+    # @param [String] output directory to save files
+    # @param [String] format format of output files (xml, yaml, bibxml)
+    #
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+      dir = File.dirname(File.expand_path(__FILE__))
+      @group_names = YAML.load_file(File.join(dir, "workgroups.yaml"))
+      @data = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
+      @files = []
+      @index = DataIndex.new
+    end
+
+    #
+    # Initialize fetcher and run fetch
+    #
+    # @param [Strin] output directory to save files, default: "data"
+    # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
+    #
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Parse documents
+    #
+    def fetch # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      query_versioned_docs.each do |sl|
+        save_doc DataParser.parse(sl, self)
+      rescue StandardError => e
+        warn "Error: document #{sl.link} #{e.message}"
+        warn e.backtrace.join("\n")
+      end
+      query_unversioned_docs.each do |sl|
+        save_doc DataParser.parse(sl, self)
+      rescue StandardError => e
+        warn "Error: document #{sl.version_of} #{e.message}"
+        warn e.backtrace.join("\n")
+      end
+      Dir[File.expand_path("../../data/*", __dir__)].each do |file|
+        xml = File.read file, encoding: "UTF-8"
+        save_doc BibXMLParser.parse(xml), warn_duplicate: false
+      rescue StandardError => e
+        warn "Error: document #{file} #{e.message}"
+        warn e.backtrace.join("\n")
+      end
+      @index.sort!.save
+    end
+
+    #
+    # Create index file
+    #
+    # def create_index
+    #   index_file = "index-w3c.yaml"
+    #   index_yaml = @index.sort do |a, b|
+    #     compare_index_items a, b
+    #   end.to_yaml
+    #   File.write index_file, index_yaml, encoding: "UTF-8"
+    # end
+
+    #
+    # Compare index items
+    #
+    # @param [Hash] aid first item
+    # @param [Hash] bid second item
+    #
+    # @return [Integer] comparison result
+    #
+    # def compare_index_items(aid, bid) # rubocop:disable Metrics/AbcSize
+    #   ret = aid[:code] <=> bid[:code]
+    #   ret = stage_weight(bid[:stage]) <=> stage_weight(aid[:stage]) if ret.zero?
+    #   ret = date_weight(bid[:date]) <=> date_weight(aid[:date]) if ret.zero?
+    #   # ret = aid[:type] <=> bid[:type] if ret.zero?
+    #   ret
+    # end
+
+    #
+    # Weight of stage
+    #
+    # @param [String, nil] stage stage
+    #
+    # @return [Integer] weight
+    #
+    # def stage_weight(stage)
+    #   return DataParser::STAGES.size if stage.nil?
+
+    #   DataParser::STAGES.keys.index(stage)
+    # end
+
+    #
+    # Weight of date
+    #
+    # @param [String] date date
+    #
+    # @return [String] weight
+    #
+    # def date_weight(date)
+    #   return "99999999" if date.nil?
+
+    #   date
+    # end
+
+    #
+    # Query RDF source for documents
+    #
+    # @return [RDF::Query::Solutions] query results
+    #
+    def query_versioned_docs # rubocop:disable Metrics/MethodLength
+      sse = SPARQL.parse(%(
+        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+        PREFIX dc: <http://purl.org/dc/elements/1.1/>
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        # PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
+        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+        SELECT ?link ?title ?date ?version_of
+        WHERE {
+          ?link dc:title ?title ; dc:date ?date ; doc:versionOf ?version_of .
+        }
+      ))
+      data.query sse
+    end
+
+    def query_unversioned_docs
+      sse = SPARQL.parse(%(
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        SELECT ?version_of
+        WHERE { ?x doc:versionOf ?version_of . }
+      ))
+      data.query(sse).uniq &:version_of
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonW3c::W3cBibliographicItem, nil] bib bibliographic item
+    #
+    def save_doc(bib, warn_duplicate: true) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      return unless bib
+
+      c = case @format
+          when "xml" then bib.to_xml(bibdata: true)
+          when "yaml" then bib.to_hash.to_yaml
+          else bib.send("to_#{@format}")
+          end
+      # id = bib.docidentifier.detect(&:primary)&.id || bib.formattedref.content
+      file = file_name(bib.docnumber)
+      if @files.include?(file)
+        warn "File #{file} already exists. Document: #{bib.docnumber}" if warn_duplicate
+      else
+        @index.add bib.docnumber, file
+        @files << file
+        File.write file, c, encoding: "UTF-8"
+      end
+    end
+
+    #
+    # Generate file name
+    #
+    # @param [String] id document id
+    #
+    # @return [String] file name
+    #
+    def file_name(id)
+      name = id.sub(/^W3C\s/, "").gsub(/[\s,:\/+]/, "_").squeeze("_").downcase
+      File.join @output, "#{name}.#{@ext}"
+    end
+  end
+end

data/lib/relaton_w3c/data_index.rb

@@ -0,0 +1,146 @@
+require "zip"
+
+module RelatonW3c
+  class DataIndex
+    #
+    # Initialize data index.
+    #
+    # @param [String] index_file path to index file
+    # @param [Array<Hash>] index index data
+    #
+    def initialize(index_file: "index-w3c.yaml", index: [])
+      @index_file = index_file
+      @index = index
+    end
+
+    #
+    # Create index from a GitHub repository
+    #
+    # @return [RelatonW3c::DataIndex] data index
+    #
+    def self.create_from_repo # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      resp = Zip::InputStream.new URI("#{W3cBibliography::SOURCE}index-w3c.zip").open
+      zip = resp.get_next_entry
+
+      # Newer versions of Psych uses the `permitted_classes:` parameter
+      index = if YAML.method(:safe_load).parameters.collect(&:last).index(:permitted_classes)
+                YAML.safe_load(zip.get_input_stream.read, permitted_classes: [Symbol])
+              else
+                YAML.safe_load(zip.get_input_stream.read, [Symbol])
+              end
+
+      DataIndex.new index: index
+    end
+
+    #
+    # Add document to index
+    #
+    # @param [String] docnumber document number
+    # @param [String] file path to document file
+    #
+    def add(docnumber, file)
+      @index << docnumber_to_parts(docnumber, file)
+    end
+
+    #
+    # Save index to file.
+    #
+    def save
+      File.write @index_file, @index.to_yaml, encoding: "UTF-8"
+    end
+
+    #
+    # Sort index
+    #
+    # @return [Array<Hash>] sorted index
+    #
+    def sort!
+      @index.sort! { |a, b| compare_index_items a, b }
+      self
+    end
+
+    #
+    # Search filename in index
+    #
+    # @param [String] ref reference
+    #
+    # @return [String] document's filename
+    #
+    def search(ref) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      dparts = docnumber_to_parts(ref)
+      @index.detect do |parts|
+        parts[:code].match?(/^#{Regexp.escape dparts[:code]}/i) &&
+          (dparts[:stage].nil? || dparts[:stage].casecmp?(parts[:stage])) &&
+          (dparts[:type].nil? || dparts[:type].casecmp?(parts[:type])) &&
+          (dparts[:date].nil? || dparts[:date] == parts[:date]) &&
+          (dparts[:suff].nil? || dparts[:suff].casecmp?(parts[:suff]))
+      end&.fetch(:file)
+    end
+
+    #
+    # Compare index items
+    #
+    # @param [Hash] aid first item
+    # @param [Hash] bid second item
+    #
+    # @return [Integer] comparison result
+    #
+    def compare_index_items(aid, bid) # rubocop:disable Metrics/AbcSize
+      ret = aid[:code] <=> bid[:code]
+      ret = stage_weight(bid[:stage]) <=> stage_weight(aid[:stage]) if ret.zero?
+      ret = date_weight(bid[:date]) <=> date_weight(aid[:date]) if ret.zero?
+      # ret = aid[:type] <=> bid[:type] if ret.zero?
+      ret
+    end
+
+    #
+    # Weight of stage
+    #
+    # @param [String, nil] stage stage
+    #
+    # @return [Integer] weight
+    #
+    def stage_weight(stage)
+      return DataParser::STAGES.size if stage.nil?
+
+      DataParser::STAGES.keys.index(stage)
+    end
+
+    #
+    # Weight of date
+    #
+    # @param [String] date date
+    #
+    # @return [String] weight
+    #
+    def date_weight(date)
+      return "99999999" if date.nil?
+
+      date
+    end
+
+    #
+    # Parse document number to parts
+    #
+    # @param [String] docnumber document number
+    # @param [String, nil] file path to document file
+    #
+    # @return [Hash{Symbol=>String}] document parts
+    #
+    def docnumber_to_parts(docnumber, file = nil) # rubocop:disable Metrics/MethodLength
+      %r{
+        ^(?:(?:(?<stage>WD|CRD|CR|PR|PER|REC|SPSD|OBSL|RET)|(?<type>D?NOTE))-)?
+        (?<code>\w+(?:[+-][\w.]+)*?)
+        (?:-(?<date>\d{8}|\d{6}))?
+        (?:/(?<suff>\w+))?$
+      }xi =~ docnumber
+      entry = { code: code }
+      entry[:file] = file if file
+      entry[:stage] = stage if stage
+      entry[:type] = type if type
+      entry[:date] = date if date
+      entry[:suff] = suff if suff
+      entry
+    end
+  end
+end

data/lib/relaton_w3c/data_parser.rb

@@ -1,5 +1,24 @@
 module RelatonW3c
   class DataParser
+    USED_TYPES = %w[WD NOTE PER PR REC CR].freeze
+
+    DOCTYPES = {
+      "TR" => "technicalReport",
+      "NOTE" => "groupNote",
+    }.freeze
+
+    STAGES = {
+      "RET" => "retired",
+      "SPSD" => "supersededRecommendation",
+      "OBSL" => "obsoletedRecommendation",
+      "WD" => "workingDraft",
+      "CRD" => "candidateRecommendationDraft",
+      "CR" => "candidateRecommendation",
+      "PR" => "proposedRecommendation",
+      "PER" => "proposedEditedRecommendation",
+      "REC" => "recommendation",
+    }.freeze
+
     #
     # Document parser initalization
     #
@@ -29,7 +48,7 @@ module RelatonW3c
     # @return [RelatonW3c:W3cBibliographicItem, nil] bibliographic item
     #
     def parse # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      return unless @fetcher.class::USED_TYPES.include? type
+      return if @sol.respond_to?(:link) && !types_stages.detect { |ts| USED_TYPES.include?(ts) }
 
       RelatonW3c::W3cBibliographicItem.new(
         type: "standard",
@@ -37,10 +56,12 @@ module RelatonW3c
         fetched: Date.today.to_s,
         language: ["en"],
         script: ["Latn"],
+        docstatus: parse_docstatus,
         title: parse_title,
         link: parse_link,
         docid: parse_docid,
-        docnumber: identifier(@sol.link.to_s),
+        formattedref: parse_formattedref,
+        docnumber: identifier,
         series: parse_series,
         date: parse_date,
         relation: parse_relation,
@@ -49,12 +70,24 @@ module RelatonW3c
       )
     end
 
+    #
+    # Extract documetn status
+    #
+    # @return [RelatonBib::DocumentStatus, nil] dcoument status
+    #
+    def parse_docstatus
+      stage = types_stages&.detect { |st| STAGES.include?(st) }
+      RelatonBib::DocumentStatus.new stage: STAGES[stage] if stage
+    end
+
     #
     # Parse title
     #
     # @return [RelatonBib::TypedTitleStringCollection] title
     #
     def parse_title
+      return [] unless @sol.respond_to?(:title)
+
       t = RelatonBib::TypedTitleString.new content: @sol.title.to_s
       RelatonBib::TypedTitleStringCollection.new [t]
     end
@@ -65,7 +98,9 @@ module RelatonW3c
     # @return [Array<RelatonBib::TypedUri>] link
     #
     def parse_link
-      [RelatonBib::TypedUri.new(type: "src", content: @sol.link.to_s)]
+      link = @sol.respond_to?(:link) ? @sol.link : @sol.version_of
+
+      [RelatonBib::TypedUri.new(type: "src", content: link.to_s)]
     end
 
     #
@@ -74,23 +109,45 @@ module RelatonW3c
     # @return [Arra<RelatonBib::DocumentIdentifier>] docidentifier
     #
     def parse_docid
-      id = pub_id(@sol.link.to_s)
+      return [] unless @sol.respond_to?(:link)
+
+      id = pub_id(@sol.link)
       [RelatonBib::DocumentIdentifier.new(type: "W3C", id: id, primary: true)]
     end
 
     #
     # Generate PubID
     #
-    # @param [String] url url
-    #
-    # @return [String] PubID
+    # @return [RDF::URI] PubID
     #
     def pub_id(url)
       "W3C #{identifier(url)}"
     end
 
-    def identifier(url)
-      /.+\/(\w+(?:-[\w.]+)+(?:\/\w+)?)/.match(url)[1].to_s
+    #
+    # Generate identifier from URL
+    #
+    # @param [RDF::URI, nil] link
+    #
+    # @return [String] identifier
+    #
+    def identifier(link = nil)
+      url = link || (@sol.respond_to?(:link) ? @sol.link : @sol.version_of)
+      self.class.parse_identifier(url.to_s)
+    end
+
+    #
+    # Parse identifier from URL
+    #
+    # @param [String] url URL
+    #
+    # @return [String] identifier
+    #
+    def self.parse_identifier(url)
+      if /.+\/(\w+(?:-[\w.]+)+(?:\/\w+)?)/ =~ url.to_s
+        $1.to_s
+      else url.to_s.split("/").last
+      end
     end
 
     #
@@ -99,12 +156,31 @@ module RelatonW3c
     # @return [Array<RelatonBib::Series>] series
     #
     def parse_series
+      return [] unless type
+
       title = RelatonBib::TypedTitleString.new content: "W3C #{type}"
-      [RelatonBib::Series.new(title: title, number: identifier(@sol.link.to_s))]
+      [RelatonBib::Series.new(title: title, number: identifier)]
     end
 
-    def type # rubocop:disable Metrics/MethodLength
-      @type ||= begin
+    #
+    # Extract type
+    #
+    # @return [String] type
+    #
+    def type
+      # thre are many types, we need to find the right one
+      @type ||= types_stages&.detect { |t| USED_TYPES.include?(t) }
+    end
+
+    #
+    # Fetches types and stages
+    #
+    # @return [Array<String>] types and stages
+    #
+    def types_stages # rubocop:disable Metrics/MethodLength
+      return unless @sol.respond_to?(:link)
+
+      @types_stages ||= begin
         sse = SPARQL.parse(%(
           PREFIX : <http://www.w3.org/2001/02pd/rec54#>
           PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
@@ -113,8 +189,7 @@ module RelatonW3c
             { <#{@sol.link}> rdf:type ?type }
           }
         ))
-        tps = @fetcher.data.query(sse).map { |s| s.type.to_s.split("#").last }
-        tps.detect { |t| Scrapper::DOCTYPES.key?(t) }
+        @fetcher.data.query(sse).map { |s| s.type.to_s.split("#").last }
       end
     end
 
@@ -124,10 +199,17 @@ module RelatonW3c
     # @return [Strinf] doctype
     #
     def parse_doctype
-      Scrapper::DOCTYPES[type]
+      DOCTYPES[type] || "recommendation"
     end
 
+    #
+    # Parse date
+    #
+    # @return [Array<RelatonBib::BibliographicDate>] date
+    #
     def parse_date
+      return [] unless @sol.respond_to?(:date)
+
       [RelatonBib::BibliographicDate.new(type: "published", on: @sol.date.to_s)]
     end
 
@@ -136,29 +218,90 @@ module RelatonW3c
     #
     # @return [Array<RelatonBib::DocumentRelation>] relation
     #
-    def parse_relation # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+    def parse_relation
+      if @sol.respond_to?(:link)
+        relations + editor_drafts
+      else document_versions
+      end
+    end
+
+    def relations # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      {
+        "doc:obsoletes" => { type: "obsoletes" },
+        "mat:hasErrata" => { type: "updatedBy", description: "errata" },
+        # "mat:hasTranslations" => "hasTranslation",
+        # "mat:hasImplReport" => "hasImpReport",
+        ":previousEdition" => { type: "editionOf" },
+      }.reduce([]) do |acc, (predicate, tp)|
+        acc + relation_query(predicate).map do |r|
+          fr = RelatonBib::LocalizedString.new pub_id(r.rel.to_s)
+          bib = W3cBibliographicItem.new formattedref: fr
+          tp[:description] = RelatonBib::FormattedString.new content: tp[:description] if tp[:description]
+          RelatonBib::DocumentRelation.new(**tp, bibitem: bib)
+        end
+      end
+    end
+
+    def editor_drafts # rubocop:disable Metrics/MethodLength
       sse = SPARQL.parse(%(
-        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
-        SELECT ?obsoletes
-        WHERE {
-          VALUES ?p { doc:obsoletes }
-          { <#{@sol.link}> ?p ?obsoletes }
-        }
+        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+        SELECT ?rel
+        WHERE { <#{@sol.link}> :ED ?rel . }
       ))
-      @fetcher.data.query(sse).order_by(:obsoletes).map do |r|
-        tp, url = r.to_h.first
-        fr = RelatonBib::LocalizedString.new pub_id(url.to_s)
+      @fetcher.data.query(sse).map do |s|
+        fr = RelatonBib::LocalizedString.new pub_id(s.rel.to_s)
         bib = W3cBibliographicItem.new formattedref: fr
-        RelatonBib::DocumentRelation.new(type: tp.to_s, bibitem: bib)
+        desc = RelatonBib::FormattedString.new content: "Editor's draft"
+        RelatonBib::DocumentRelation.new(
+          type: "hasDraft", description: desc, bibitem: bib,
+        )
+      end
+    end
+
+    def relation_query(predicate)
+      sse = SPARQL.parse(%(
+        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
+        SELECT ?rel
+        WHERE { <#{@sol.link}> #{predicate} ?rel . }
+      ))
+      @fetcher.data.query(sse).order_by(:rel)
+    end
+
+    def document_versions
+      sse = SPARQL.parse(%(
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        SELECT ?link
+        WHERE { ?link doc:versionOf <#{@sol.version_of}> }
+      ))
+      @fetcher.data.query(sse).map do |r|
+        fref = RelatonBib::FormattedRef.new content: pub_id(r.link)
+        bib = W3cBibliographicItem.new formattedref: fref
+        RelatonBib::DocumentRelation.new(type: "hasEdition", bibitem: bib)
       end
     end
 
+    #
+    # Parse formattedref
+    #
+    # @return [RelatonBib::FormattedRef] formattedref
+    #
+    def parse_formattedref
+      return if @sol.respond_to?(:link)
+
+      RelatonBib::FormattedRef.new(content: pub_id(@sol.version_of))
+    end
+
     #
     # Parse contributor
     #
     # @return [Array<RelatonBib::ContributionInfo>] contributor
     #
     def parse_contrib # rubocop:disable Metrics/MethodLength
+      return [] unless @sol.respond_to?(:link)
+
       sse = SPARQL.parse(%(
         PREFIX : <http://www.w3.org/2001/02pd/rec54#>
         PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>
@@ -181,6 +324,8 @@ module RelatonW3c
     # @return [RelatonBib::EditorialGroup] editorialgroup
     #
     def parse_editorialgroup # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      return unless @sol.respond_to?(:link)
+
       sse = SPARQL.parse(%(
         PREFIX org: <http://www.w3.org/2001/04/roadmap/org#>
         PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>

data/lib/relaton_w3c/version.rb

@@ -1,3 +1,3 @@
 module RelatonW3c
-  VERSION = "1.11.0".freeze
+  VERSION = "1.11.3".freeze
 end

data/lib/relaton_w3c/w3c_bibliography.rb

@@ -5,15 +5,17 @@ require "net/http"
 module RelatonW3c
   # Class methods for search W3C standards.
   class W3cBibliography
-    SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-w3c/main/data/"
+    SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-w3c/main/"
 
     class << self
       # @param text [String]
       # @return [RelatonW3c::HitCollection]
       def search(text) # rubocop:disable Metrics/MethodLength
-        # HitCollection.new text
-        file = text.sub(/^W3C\s/, "").gsub(/[\s,:\/]/, "_").squeeze("_").upcase
-        url = "#{SOURCE}#{file}.yaml"
+        ref = DataParser.parse_identifier text.sub(/^W3C\s/, "")
+        file = DataIndex.create_from_repo.search(ref)
+        return unless file
+
+        url = "#{SOURCE}#{file}"
         resp = Net::HTTP.get_response(URI.parse(url))
         return unless resp.code == "200"
 
@@ -24,7 +26,7 @@ module RelatonW3c
              EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
              Net::ProtocolError, Errno::ETIMEDOUT
         raise RelatonBib::RequestError,
-              "Could not access #{HitCollection::DOMAIN}"
+              "Could not access #{url}"
       end
 
       # @param ref [String] the W3C standard Code to look up
@@ -39,8 +41,8 @@ module RelatonW3c
           return
         end
 
-        # ret = result.first.fetch
-        warn "[relaton-w3c] (\"#{ref}\") found #{result.title.first.title.content}"
+        found = result.docnumber
+        warn "[relaton-w3c] (\"#{ref}\") found #{found}"
         result
       end
     end

data/lib/relaton_w3c/workgroups.yaml

@@ -32,6 +32,9 @@
 'https://www.w3.org/WAI/EO':
   name: Education and Outreach Working Group 
   abbrev: EOWG
+'https://www.w3.org/WAI/about/groups/eowg':
+  name: Education and Outreach Working Group 
+  abbrev: EOWG
 'https://www.w3.org/2001/sw/WebOnt':
   name: Web-Ontology Working Group
 'http://www.w3.org/MarkUp/Forms':
@@ -54,6 +57,8 @@
   name: Web Applications Working Group
 'https://www.w3.org/2008/webapps':
   name: Web Applications Working Group
+'https://www.w3.org/groups/wg/webapps':
+  name: Web Applications Working Group
 'https://www.w3.org/das':
   name: Devices and Sensors Working Group
   abbrev: DAS WG
@@ -226,6 +231,8 @@
   abbrev: ARIA WG
 'https://www.w3.org/wasm':
   name: WebAssembly Working Group
+'https://www.w3.org/groups/wg/wasm':
+  name: WebAssembly Working Group
 'https://www.w3.org/groups/wg/webediting':
   name: Web Editing Working Group
 'https://www.w3.org/2014/data-shapes':

data/lib/relaton_w3c.rb

@@ -2,13 +2,14 @@ require "relaton_bib"
 require "relaton_w3c/version"
 require "relaton_w3c/w3c_bibliography"
 require "relaton_w3c/w3c_bibliographic_item"
-require "relaton_w3c/hit_collection"
-require "relaton_w3c/hit"
-require "relaton_w3c/scrapper"
+# require "relaton_w3c/hit_collection"
+# require "relaton_w3c/hit"
+# require "relaton_w3c/scrapper"
 require "relaton_w3c/xml_parser"
 require "relaton_w3c/bibxml_parser"
 require "relaton_w3c/hash_converter"
-require "relaton_w3c/data_fethcer"
+require "relaton_w3c/data_fetcher"
+require "relaton_w3c/data_index"
 
 module RelatonW3c
   class Error < StandardError; end

data/relaton_w3c.gemspec

@@ -40,6 +40,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "linkeddata", "~> 3.1.0"
   spec.add_dependency "mechanize", "~> 2.8.0"
   spec.add_dependency "rdf", "~> 3.1.0"
+  spec.add_dependency "rdf-normalize", "~> 0.4.0"
   spec.add_dependency "relaton-bib", "~> 1.11.0"
+  spec.add_dependency "rubyzip", "~> 2.3.0"
+  spec.add_dependency "shex", "~> 0.6.0"
   spec.add_dependency "sparql", "~> 3.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-w3c
 version: !ruby/object:Gem::Version
-  version: 1.11.0
+  version: 1.11.3
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-04-10 00:00:00.000000000 Z
+date: 2022-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: rdf-normalize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +150,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.11.0
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+- !ruby/object:Gem::Dependency
+  name: shex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: sparql
   requirement: !ruby/object:Gem::Requirement
@@ -201,7 +243,8 @@ files:
 - grammars/reqt.rng
 - lib/relaton_w3c.rb
 - lib/relaton_w3c/bibxml_parser.rb
-- lib/relaton_w3c/data_fethcer.rb
+- lib/relaton_w3c/data_fetcher.rb
+- lib/relaton_w3c/data_index.rb
 - lib/relaton_w3c/data_parser.rb
 - lib/relaton_w3c/hash_converter.rb
 - lib/relaton_w3c/hit.rb

data/lib/relaton_w3c/data_fethcer.rb

@@ -1,110 +0,0 @@
-require "rdf"
-require "linkeddata"
-require "sparql"
-require "mechanize"
-require "relaton_w3c/data_parser"
-
-module RelatonW3c
-  class DataFetcher
-    USED_TYPES = %w[WD NOTE PER PR REC CR].freeze
-
-    attr_reader :data, :group_names
-
-    #
-    # Data fetcher initializer
-    #
-    # @param [String] output directory to save files
-    # @param [String] format format of output files (xml, yaml, bibxml)
-    #
-    def initialize(output, format)
-      @output = output
-      @format = format
-      @ext = format.sub(/^bib/, "")
-      dir = File.dirname(File.expand_path(__FILE__))
-      @group_names = YAML.load_file(File.join(dir , "workgroups.yaml"))
-      @data = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
-      @files = []
-    end
-
-    #
-    # Initialize fetcher and run fetch
-    #
-    # @param [Strin] output directory to save files, default: "data"
-    # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
-    #
-    def self.fetch(output: "data", format: "yaml")
-      t1 = Time.now
-      puts "Started at: #{t1}"
-      FileUtils.mkdir_p output unless Dir.exist? output
-      new(output, format).fetch
-      t2 = Time.now
-      puts "Stopped at: #{t2}"
-      puts "Done in: #{(t2 - t1).round} sec."
-    end
-
-    #
-    # Parse documents
-    #
-    def fetch
-      query.each { |sl| save_doc DataParser.parse(sl, self) }
-      Dir[File.expand_path("../../data/*", __dir__)].each do |file|
-        xml = File.read file, encoding: "UTF-8"
-        save_doc BibXMLParser.parse(xml)
-      end
-    end
-
-    #
-    # Query RDF source for documents
-    #
-    # @return [RDF::Query::Solutions] query results
-    #
-    def query # rubocop:disable Metrics/MethodLength
-      sse = SPARQL.parse(%(
-        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
-        PREFIX dc: <http://purl.org/dc/elements/1.1/>
-        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
-        # PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
-        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
-        SELECT ?link ?title ?date
-        WHERE {
-          ?link dc:title ?title ; dc:date ?date . # ; doc:versionOf ?version_of .
-        }
-      ))
-      data.query sse
-    end
-
-    #
-    # Save document to file
-    #
-    # @param [RelatonW3c::W3cBibliographicItem, nil] bib bibliographic item
-    #
-    def save_doc(bib) # rubocop:disable Metrics/MethodLength
-      return unless bib
-
-      c = case @format
-          when "xml" then bib.to_xml(bibdata: true)
-          when "yaml" then bib.to_hash.to_yaml
-          else bib.send("to_#{@format}")
-          end
-      file = file_name(bib)
-      if @files.include? file
-        warn "File #{file} already exists. Document: #{bib.docnumber}"
-      else
-        @files << file
-      end
-      File.write file, c, encoding: "UTF-8"
-    end
-
-    #
-    # Generate file name
-    #
-    # @param [RelatonW3c::W3cBibliographicItem] bib bibliographic item
-    #
-    # @return [String] file name
-    #
-    def file_name(bib)
-      name = bib.docnumber.gsub(/[\s,:\/]/, "_").squeeze("_").upcase
-      File.join @output, "#{name}.#{@ext}"
-    end
-  end
-end