relaton-w3c 1.11.5 → 1.12.1

data/lib/relaton_w3c/data_fetcher.rb

@@ -20,22 +20,21 @@ module RelatonW3c
       @ext = format.sub(/^bib/, "")
       dir = File.dirname(File.expand_path(__FILE__))
       @group_names = YAML.load_file(File.join(dir, "workgroups.yaml"))
-      @data = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
-      @files = []
       @index = DataIndex.new
     end
 
     #
     # Initialize fetcher and run fetch
     #
+    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
     # @param [Strin] output directory to save files, default: "data"
     # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
     #
-    def self.fetch(output: "data", format: "yaml")
+    def self.fetch(source, output: "data", format: "yaml")
       t1 = Time.now
       puts "Started at: #{t1}"
-      FileUtils.mkdir_p output unless Dir.exist? output
-      new(output, format).fetch
+      FileUtils.mkdir_p output
+      new(output, format).fetch source
       t2 = Time.now
       puts "Stopped at: #{t2}"
       puts "Done in: #{(t2 - t1).round} sec."
@@ -44,19 +43,89 @@ module RelatonW3c
     #
     # Parse documents
     #
-    def fetch # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      query_versioned_docs.each do |sl|
-        save_doc DataParser.parse(sl, self)
-      rescue StandardError => e
-        warn "Error: document #{sl.link} #{e.message}"
-        warn e.backtrace.join("\n")
+    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
+    #
+    def fetch(source) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      each_dataset(source) do |rdf|
+        %i[versioned unversioned].each do |type|
+          send("query_#{type}_docs", rdf).each do |sl|
+            bib = DataParser.parse(rdf, sl, self)
+            add_has_edition_relation(bib) if type == :unversioned
+            save_doc bib
+          rescue StandardError => e
+            link = sl.respond_to?(:link) ? sl.link : sl.version_of
+            warn "Error: document #{link} #{e.message}"
+            warn e.backtrace.join("\n")
+          end
+        end
       end
-      query_unversioned_docs.each do |sl|
-        save_doc DataParser.parse(sl, self)
-      rescue StandardError => e
-        warn "Error: document #{sl.version_of} #{e.message}"
-        warn e.backtrace.join("\n")
+      @index.sort!.save
+    end
+
+    #
+    # Add hasEdition relations form previous parsed document
+    #
+    # @param [RelatonW3c::W3cBibliographicItem] bib bibligraphic item
+    #
+    def add_has_edition_relation(bib) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity
+      file = file_name bib.docnumber
+      return unless File.exist? file
+
+      b = case @format
+          when "xml" then XMLParser.from_xml(File.read(file, encoding: "UTF-8"))
+          when "yaml"
+            hash = YAML.load_file(file)
+            W3cBibliographicItem.from_hash(hash)
+          when "bibxml" then BibXMLParser.parse File.read(file, encoding: "UTF-8")
+          end
+      b.relation.each do |r|
+        same_edition = bib.relation.detect { |r2| same_edition?(r, r2) }
+        bib.relation << r unless same_edition
       end
+    end
+
+    #
+    # Compare two relations
+    #
+    # @param [RelatonW3c::W3cBibliographicItem] rel1 relation 1
+    # @param [RelatonW3c::W3cBibliographicItem] rel2 relation 2
+    #
+    # @return [Boolean] true if relations are same
+    #
+    def same_edition?(rel1, rel2)
+      return false unless rel1.type == "hasEdition" && rel1.type == rel2.type
+
+      ids1 = rel1.bibitem.docidentifier.map(&:id)
+      ids2 = rel2.bibitem.docidentifier.map(&:id)
+      (ids1 & ids2).any?
+    end
+
+    #
+    # Yield fetching for each dataset
+    #
+    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
+    #
+    # @yield [RDF::Repository] RDF repository
+    #
+    def each_dataset(source, &_block) # rubocop:disable Metrics/MethodLength
+      case source
+      when "w3c-tr-archive"
+        Dir["w3c-tr-archive/*.rdf"].map do |f|
+          @files = []
+          yield RDF::Repository.load(f)
+        end
+      when "w3c-rdf"
+        @files = []
+        rdf = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
+        yield rdf
+        parse_static_dataset
+      end
+    end
+
+    #
+    # Parse static dataset
+    #
+    def parse_static_dataset
       Dir[File.expand_path("../../data/*", __dir__)].each do |file|
         xml = File.read file, encoding: "UTF-8"
         save_doc BibXMLParser.parse(xml), warn_duplicate: false
@@ -64,36 +133,40 @@ module RelatonW3c
         warn "Error: document #{file} #{e.message}"
         warn e.backtrace.join("\n")
       end
-      @index.sort!.save
     end
 
     #
-    # Query RDF source for documents
+    # Query RDF source for versioned documents
     #
     # @return [RDF::Query::Solutions] query results
     #
-    def query_versioned_docs # rubocop:disable Metrics/MethodLength
+    def query_versioned_docs(rdf)
       sse = SPARQL.parse(%(
         PREFIX : <http://www.w3.org/2001/02pd/rec54#>
         PREFIX dc: <http://purl.org/dc/elements/1.1/>
         PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
-        # PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
         PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
-        SELECT ?link ?title ?date ?version_of
-        WHERE {
-          ?link dc:title ?title ; dc:date ?date ; doc:versionOf ?version_of .
-        }
+        SELECT ?link ?title ?date
+        WHERE { ?link dc:title ?title ; dc:date ?date . }
       ))
-      data.query sse
+      rdf.query sse
     end
 
-    def query_unversioned_docs
+    #
+    # Query RDF source for unversioned documents
+    #
+    # @return [Array<RDF::Query::Solution>] query results
+    #
+    def query_unversioned_docs(rdf)
       sse = SPARQL.parse(%(
         PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
         SELECT ?version_of
-        WHERE { ?x doc:versionOf ?version_of . }
+        WHERE {
+          ?link doc:versionOf ?version_of .
+          FILTER ( isURI(?link) && isURI(?version_of) && ?link != ?version_of )
+        }
       ))
-      data.query(sse).uniq &:version_of
+      rdf.query(sse).uniq { |s| s.version_of.to_s.sub(/^https?:\/\//, "").sub(/\/$/, "") }
     end
 
     #

data/lib/relaton_w3c/data_index.rb

@@ -14,15 +14,24 @@ module RelatonW3c
     end
 
     #
-    # Add document to index
+    # Add document to index or update it if already exists
     #
     # @param [String] docnumber document number
     # @param [String] file path to document file
     #
-    def add(docnumber, file)
+    def add(docnumber, file) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
       dnparts = self.class.docnumber_to_parts docnumber
-      dnparts[:file] = file
-      @index << dnparts
+      rec = @index.detect { |i| i[:file] == file }
+      if rec
+        rec[:code] = dnparts[:code]
+        dnparts[:stage] ? rec[:stage] = dnparts[:stage] : rec.delete(:stage)
+        dnparts[:type] ? rec[:type] = dnparts[:type] : rec.delete(:type)
+        dnparts[:date] ? rec[:date] = dnparts[:date] : rec.delete(:date)
+        dnparts[:suff] ? rec[:suff] = dnparts[:suff] : rec.delete(:suff)
+      else
+        dnparts[:file] = file
+        @index << dnparts
+      end
     end
 
     #
@@ -111,18 +120,24 @@ module RelatonW3c
       #
       # @return [RelatonW3c::DataIndex] data index
       #
-      def create_from_repo # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-        resp = Zip::InputStream.new URI("#{W3cBibliography::SOURCE}index-w3c.zip").open
+      def create_from_repo
+        uri = URI("#{W3cBibliography::SOURCE}index-w3c.zip").open
+        resp = Zip::InputStream.new uri
         zip = resp.get_next_entry
+        index = RelatonBib.parse_yaml(zip.get_input_stream.read, [Symbol])
+        new index: index
+      end
 
-        # Newer versions of Psych uses the `permitted_classes:` parameter
-        index = if YAML.method(:safe_load).parameters.collect(&:last).index(:permitted_classes)
-                  YAML.safe_load(zip.get_input_stream.read, permitted_classes: [Symbol])
-                else
-                  YAML.safe_load(zip.get_input_stream.read, [Symbol])
-                end
-
-        DataIndex.new index: index
+      #
+      # Create index from a file
+      #
+      # @param [String] index_file path to index file
+      #
+      # @return [RelatonW3c::DataIndex] data index
+      #
+      def create_from_file(index_file = "index-w3c.yaml")
+        index = RelatonBib.parse_yaml(File.read(index_file), [Symbol])
+        new index_file: index_file, index: index
       end
 
       #

data/lib/relaton_w3c/data_parser.rb

@@ -25,7 +25,8 @@ module RelatonW3c
     # @param [RDF::Query::Solution] sol entry from the SPARQL query
     # @param [RelatonW3c::DataFetcher] fetcher data fetcher
     #
-    def initialize(sol, fetcher)
+    def initialize(rdf, sol, fetcher)
+      @rdf = rdf
       @sol = sol
       @fetcher = fetcher
     end
@@ -38,8 +39,8 @@ module RelatonW3c
     #
     # @return [RelatonW3c:W3cBibliographicItem, nil] bibliographic item
     #
-    def self.parse(sol, fetcher)
-      new(sol, fetcher).parse
+    def self.parse(rdf, sol, fetcher)
+      new(rdf, sol, fetcher).parse
     end
 
     #
@@ -86,9 +87,10 @@ module RelatonW3c
     # @return [RelatonBib::TypedTitleStringCollection] title
     #
     def parse_title
-      return [] unless @sol.respond_to?(:title)
-
-      t = RelatonBib::TypedTitleString.new content: @sol.title.to_s
+      content = if @sol.respond_to?(:title) then @sol.title.to_s
+                else document_versions.max_by { |dv| dv.date.to_s }.title.to_s
+                end
+      t = RelatonBib::TypedTitleString.new content: content
       RelatonBib::TypedTitleStringCollection.new [t]
     end
 
@@ -99,8 +101,7 @@ module RelatonW3c
     #
     def parse_link
       link = @sol.respond_to?(:link) ? @sol.link : @sol.version_of
-
-      [RelatonBib::TypedUri.new(type: "src", content: link.to_s)]
+      [RelatonBib::TypedUri.new(type: "src", content: link.to_s.strip)]
     end
 
     #
@@ -109,9 +110,7 @@ module RelatonW3c
     # @return [Arra<RelatonBib::DocumentIdentifier>] docidentifier
     #
     def parse_docid
-      return [] unless @sol.respond_to?(:link)
-
-      id = pub_id(@sol.link)
+      id = @sol.respond_to?(:link) ? pub_id(@sol.link) : pub_id(@sol.version_of)
       [RelatonBib::DocumentIdentifier.new(type: "W3C", id: id, primary: true)]
     end
 
@@ -133,7 +132,7 @@ module RelatonW3c
     #
     def identifier(link = nil)
       url = link || (@sol.respond_to?(:link) ? @sol.link : @sol.version_of)
-      self.class.parse_identifier(url.to_s)
+      self.class.parse_identifier(url.to_s.strip)
     end
 
     #
@@ -144,7 +143,7 @@ module RelatonW3c
     # @return [String] identifier
     #
     def self.parse_identifier(url)
-      if /.+\/(\w+(?:-[\w.]+)+(?:\/\w+)?)/ =~ url.to_s
+      if /.+\/(\w+(?:[-+][\w.]+)+(?:\/\w+)?)/ =~ url.to_s
         $1.to_s
       else url.to_s.split("/").last
       end
@@ -186,10 +185,10 @@ module RelatonW3c
           PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
           SELECT ?type
           WHERE {
-            { <#{@sol.link}> rdf:type ?type }
+            { <#{@sol.link.to_s.strip}> rdf:type ?type }
           }
         ))
-        @fetcher.data.query(sse).map { |s| s.type.to_s.split("#").last }
+        @rdf.query(sse).map { |s| s.type.to_s.split("#").last }
       end
     end
 
@@ -221,10 +220,16 @@ module RelatonW3c
     def parse_relation
       if @sol.respond_to?(:link)
         relations + editor_drafts
-      else document_versions
+      else
+        document_versions.map { |r| create_relation(r.link.to_s.strip, "hasEdition") }
       end
     end
 
+    #
+    # Create relations
+    #
+    # @return [Array<RelatonBib::DocumentRelation>] relations
+    #
     def relations # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       {
         "doc:obsoletes" => { type: "obsoletes" },
@@ -234,53 +239,104 @@ module RelatonW3c
         ":previousEdition" => { type: "editionOf" },
       }.reduce([]) do |acc, (predicate, tp)|
         acc + relation_query(predicate).map do |r|
-          fr = RelatonBib::LocalizedString.new pub_id(r.rel.to_s)
-          bib = W3cBibliographicItem.new formattedref: fr
-          tp[:description] = RelatonBib::FormattedString.new content: tp[:description] if tp[:description]
-          RelatonBib::DocumentRelation.new(**tp, bibitem: bib)
+          create_relation(r.rel.to_s, tp[:type], tp[:description])
         end
       end
     end
 
+    #
+    # Parse editor drafts relation
+    #
+    # @return [Array<RelatonBib::DocumentRelation>] relation
+    #
     def editor_drafts # rubocop:disable Metrics/MethodLength
       sse = SPARQL.parse(%(
         PREFIX : <http://www.w3.org/2001/02pd/rec54#>
         PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
         SELECT ?rel
-        WHERE { <#{@sol.link}> :ED ?rel . }
+        WHERE { <#{@sol.link.to_s.strip}> :ED ?rel . }
       ))
-      @fetcher.data.query(sse).map do |s|
-        fr = RelatonBib::LocalizedString.new pub_id(s.rel.to_s)
-        bib = W3cBibliographicItem.new formattedref: fr
-        desc = RelatonBib::FormattedString.new content: "Editor's draft"
-        RelatonBib::DocumentRelation.new(
-          type: "hasDraft", description: desc, bibitem: bib,
-        )
+      @rdf.query(sse).map do |s|
+        create_relation(s.rel.to_s, "hasDraft", "Editor's draft")
       end
     end
 
+    #
+    # Query for relations
+    #
+    # @param [String] predicate relation type
+    #
+    # @return [RDF::Query::Solutions] query result
+    #
     def relation_query(predicate)
       sse = SPARQL.parse(%(
         PREFIX : <http://www.w3.org/2001/02pd/rec54#>
         PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
         PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
         SELECT ?rel
-        WHERE { <#{@sol.link}> #{predicate} ?rel . }
+        WHERE { <#{@sol.link.to_s.strip}> #{predicate} ?rel . }
       ))
-      @fetcher.data.query(sse).order_by(:rel)
+      @rdf.query(sse).order_by(:rel)
+    end
+
+    #
+    # Query document versions relations
+    #
+    # @return [Array<RDF::Query::Solution>] query results
+    #
+    def document_versions # rubocop:disable Metrics/MethodLength
+      @document_versions ||= version_of.each_with_object([]) do |s, acc|
+        sse = SPARQL.parse(%(
+          PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+          PREFIX dc: <http://purl.org/dc/elements/1.1/>
+          PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+          PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+          SELECT ?link ?title ?date
+          WHERE {
+            ?link doc:versionOf <#{s.version_of}> ;
+            dc:title ?title ;
+            dc:date ?date .
+          }
+        ))
+        @rdf.query(sse).each { |r| acc << r }
+      end
     end
 
-    def document_versions
+    #
+    # Query for document versions
+    #
+    # @return [RDF::Query::Solutions] query results
+    #
+    def version_of
+      return [@sol] unless @sol.respond_to?(:link)
+
       sse = SPARQL.parse(%(
         PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
-        SELECT ?link
-        WHERE { ?link doc:versionOf <#{@sol.version_of}> }
+        SELECT ?version_of
+        WHERE {
+          <#{@sol.link.to_s.strip}> doc:versionOf ?version_of .
+          FILTER ( isURI(?version_of) && <#{@sol.link.to_s.strip}> != str(?version_of) )
+        }
       ))
-      @fetcher.data.query(sse).map do |r|
-        fref = RelatonBib::FormattedRef.new content: pub_id(r.link)
-        bib = W3cBibliographicItem.new formattedref: fref
-        RelatonBib::DocumentRelation.new(type: "hasEdition", bibitem: bib)
-      end
+      @rdf.query(sse)
+    end
+
+    #
+    # Create relation
+    #
+    # @param [String] url relation URL
+    # @param [String] type relation type
+    # @param [String, nil] desc relation description
+    #
+    # @return [RelatonBib::DocumentRelation] <description>
+    #
+    def create_relation(url, type, desc = nil)
+      id = pub_id(url)
+      fref = RelatonBib::FormattedRef.new content: id
+      docid = RelatonBib::DocumentIdentifier.new(type: "W3C", id: id, primary: true)
+      bib = W3cBibliographicItem.new formattedref: fref, docid: [docid]
+      dsc = RelatonBib::FormattedString.new content: desc if desc
+      RelatonBib::DocumentRelation.new(type: type, bibitem: bib, description: dsc)
     end
 
     #
@@ -307,10 +363,10 @@ module RelatonW3c
         PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>
         SELECT ?full_name
         WHERE {
-          <#{@sol.link}> :editor/contact:fullName ?full_name
+          <#{@sol.link.to_s.strip}> :editor/contact:fullName ?full_name
         }
       ))
-      @fetcher.data.query(sse).order_by(:full_name).map do |ed|
+      @rdf.query(sse).order_by(:full_name).map do |ed|
         cn = RelatonBib::LocalizedString.new(ed.full_name.to_s, "en", "Latn")
         n = RelatonBib::FullName.new completename: cn
         p = RelatonBib::Person.new name: n
@@ -331,12 +387,13 @@ module RelatonW3c
         PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>
         SELECT ?home_page
         WHERE {
-          <#{@sol.link}> org:deliveredBy/contact:homePage ?home_page
+          <#{@sol.link.to_s.strip}> org:deliveredBy/contact:homePage ?home_page
         }
       ))
-      res = @fetcher.data.query(sse).order_by(:home_page)
+      res = @rdf.query(sse).order_by(:home_page)
       tc = res.each_with_object([]) do |edg, obj|
-        wg = @fetcher.group_names[edg.home_page.to_s.sub(/\/$/, "")]
+        group_path = edg.home_page.to_s.sub(/^https?:\/\//, "").sub(/\/$/, "")
+        wg = @fetcher.group_names[group_path]
         if wg
           rwg = RelatonBib::WorkGroup.new name: wg["name"]
           obj << RelatonBib::TechnicalCommittee.new(rwg)

data/lib/relaton_w3c/processor.rb

@@ -9,7 +9,7 @@ module RelatonW3c
       @prefix = "W3C"
       @defaultprefix = %r{^W3C\s}
       @idtype = "W3C"
-      @datasets = %w[w3c-rdf]
+      @datasets = %w[w3c-rdf w3c-tr-archive]
     end
 
     # @param code [String]
@@ -28,8 +28,8 @@ module RelatonW3c
     # @option opts [String] :output directory to output documents
     # @option opts [String] :format
     #
-    def fetch_data(_source, opts)
-      DataFetcher.fetch(**opts)
+    def fetch_data(source, opts)
+      DataFetcher.fetch(source, **opts)
     end
 
     # @param xml [String]

data/lib/relaton_w3c/version.rb

@@ -1,3 +1,3 @@
 module RelatonW3c
-  VERSION = "1.11.5".freeze
+  VERSION = "1.12.1".freeze
 end