relaton-w3c 1.12.0 → 1.12.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3bac445b656e3c97ea5e550a1c0d93f41e31a50adb5e04071e95076f139ba35d
-  data.tar.gz: bc122a2538c95e1b613f7939de6b6c7e33d98c8b0d1facf48ce94d9fd0dece06
+  metadata.gz: b0e1e914bfe536b10f9d3b9e48f946e9d86a66df1db0233e1b6d3f67b29beaa6
+  data.tar.gz: 635cd920c1b8ff93678083790518eade6ce7051653e190eb8294a25306e3e69a
 SHA512:
-  metadata.gz: b2055507b27e70a521e725368d48e063ca3cc2f4d86c84d1476dcac6cda71f06af4ad67c70f466c0b205864d90c704a8084e617054cee1bc4f419a0745d22e1d
-  data.tar.gz: 9d7268f28268075c41af992ad40e7769c7a77fb0591e6950f1dee981dc3f1fcaff76821c9cc0d5a8a2508a7390ee8f03b5e61ed8fee77ccc7485717b534d1d88
+  metadata.gz: bcdce1b3f1e10def99be7e5d51d98acb868d9d94884211d17814b22ffc64b5d2b4cfdee45bebe784fe02672d1af0cabb1a7dc7f3671433bd63c296a9ed106016
+  data.tar.gz: 81457b4d1f0f59a6b869a7089bd9173164abd126ed17a923f5f02f1766e7d732ba421c61284cfd35b1ca006b75c0e8e1ff7e80d55f58d7af09f64460bd8e1d8f

data/README.adoc

@@ -151,14 +151,17 @@ RelatonW3c::W3cBibliographicItem.new **bib_hash
 
 === Fetch data
 
-There is a W3C dataset http://www.w3.org/2002/01/tr-automation/tr.rdf which can be converted into RelatonXML/BibXML/BibYAML formats. The static files from the `path_to_gem/withdrawn` directory are added to output during fetching data.
-
-The method `RelatonW3c::DataFetcher.fetch(output: "data", format: "yaml")` converts all the documents from the dataset and save them to the `./data` folder in YAML format.
+The method `RelatonW3c::DataFetcher.fetch(source, output: "data", format: "yaml")` converts all the documents from the dataset and save them to the `./data` folder in YAML format.
 Arguments:
 
+- `source` - name of dataset (`w3c-rdf` or `w3c-tr-archive`)
 - `output` - folder to save documents (default './data').
 - `format` - format in which the documents are saved. Possible formats are: `yaml`, `xml`, `bibxml` (default `yaml`).
 
+The available datasets are:
+- `w3c-rdf` - The dataset is fetched from http://www.w3.org/2002/01/tr-automation/tr.rdf.
+- `w3c-tr-archive` - The archive dataset files should be downloaded from https://github.com/relaton/w3c-tr-archive repository and placed into `w3c-tr-archive` folder.
+
 [source,ruby]
 ----
 RelatonW3c::DataFetcher.fetch

data/lib/relaton_w3c/data_fetcher.rb

@@ -20,22 +20,21 @@ module RelatonW3c
       @ext = format.sub(/^bib/, "")
       dir = File.dirname(File.expand_path(__FILE__))
       @group_names = YAML.load_file(File.join(dir, "workgroups.yaml"))
-      @data = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
-      @files = []
-      @index = DataIndex.new
+      @index = DataIndex.create_from_file
     end
 
     #
     # Initialize fetcher and run fetch
     #
+    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
     # @param [Strin] output directory to save files, default: "data"
     # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
     #
-    def self.fetch(output: "data", format: "yaml")
+    def self.fetch(source, output: "data", format: "yaml")
       t1 = Time.now
       puts "Started at: #{t1}"
-      FileUtils.mkdir_p output unless Dir.exist? output
-      new(output, format).fetch
+      FileUtils.mkdir_p output
+      new(output, format).fetch source
       t2 = Time.now
       puts "Stopped at: #{t2}"
       puts "Done in: #{(t2 - t1).round} sec."
@@ -44,47 +43,113 @@ module RelatonW3c
     #
     # Parse documents
     #
-    def fetch # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      query_versioned_docs.each do |sl|
-        save_doc DataParser.parse(sl, self)
-      rescue StandardError => e
-        warn "Error: document #{sl.link} #{e.message}"
-        warn e.backtrace.join("\n")
+    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
+    #
+    def fetch(source) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      each_dataset(source) do |rdf|
+        %i[versioned unversioned].each do |type|
+          send("query_#{type}_docs", rdf).each do |sl|
+            bib = DataParser.parse(rdf, sl, self)
+            add_has_edition_relation(bib) if type == :unversioned
+            save_doc bib
+          rescue StandardError => e
+            link = sl.respond_to?(:link) ? sl.link : sl.version_of
+            warn "Error: document #{link} #{e.message}"
+            warn e.backtrace.join("\n")
+          end
+        end
       end
-      query_unversioned_docs.each do |sl|
-        save_doc DataParser.parse(sl, self)
-      rescue StandardError => e
-        warn "Error: document #{sl.version_of} #{e.message}"
-        warn e.backtrace.join("\n")
+      @index.sort!.save
+    end
+
+    #
+    # Add hasEdition relations form previous parsed document
+    #
+    # @param [RelatonW3c::W3cBibliographicItem] bib bibligraphic item
+    #
+    def add_has_edition_relation(bib) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity
+      file = file_name bib.docnumber
+      return unless File.exist? file
+
+      b = case @format
+          when "xml" then XMLParser.from_xml(File.read(file, encoding: "UTF-8"))
+          when "yaml"
+            hash = YAML.load_file(file)
+            W3cBibliographicItem.from_hash(hash)
+          when "bibxml" then BibXMLParser.parse File.read(file, encoding: "UTF-8")
+          end
+      b.relation.each do |r|
+        same_edition = bib.relation.detect { |r2| same_edition?(r, r2) }
+        bib.relation << r unless same_edition
       end
-      Dir[File.expand_path("../../data/*", __dir__)].each do |file|
-        xml = File.read file, encoding: "UTF-8"
-        save_doc BibXMLParser.parse(xml), warn_duplicate: false
-      rescue StandardError => e
-        warn "Error: document #{file} #{e.message}"
-        warn e.backtrace.join("\n")
+    end
+
+    #
+    # Compare two relations
+    #
+    # @param [RelatonW3c::W3cBibliographicItem] rel1 relation 1
+    # @param [RelatonW3c::W3cBibliographicItem] rel2 relation 2
+    #
+    # @return [Boolean] true if relations are same
+    #
+    def same_edition?(rel1, rel2)
+      return false unless rel1.type == "hasEdition" && rel1.type == rel2.type
+
+      ids1 = rel1.bibitem.docidentifier.map(&:id)
+      ids2 = rel2.bibitem.docidentifier.map(&:id)
+      (ids1 & ids2).any?
+    end
+
+    #
+    # Yield fetching for each dataset
+    #
+    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
+    #
+    # @yield [RDF::Repository] RDF repository
+    #
+    def each_dataset(source, &_block) # rubocop:disable Metrics/MethodLength
+      case source
+      when "w3c-tr-archive"
+        Dir["w3c-tr-archive/*.rdf"].map do |f|
+          @files = []
+          yield RDF::Repository.load(f)
+        end
+      when "w3c-rdf"
+        @files = []
+        rdf = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
+        yield rdf
+        # parse_static_dataset
       end
-      @index.sort!.save
     end
 
+    #
+    # Parse static dataset
+    #
+    # def parse_static_dataset
+    #   Dir[File.expand_path("../../data/*", __dir__)].each do |file|
+    #     xml = File.read file, encoding: "UTF-8"
+    #     save_doc BibXMLParser.parse(xml), warn_duplicate: false
+    #   rescue StandardError => e
+    #     warn "Error: document #{file} #{e.message}"
+    #     warn e.backtrace.join("\n")
+    #   end
+    # end
+
     #
     # Query RDF source for versioned documents
     #
     # @return [RDF::Query::Solutions] query results
     #
-    def query_versioned_docs # rubocop:disable Metrics/MethodLength
+    def query_versioned_docs(rdf)
       sse = SPARQL.parse(%(
         PREFIX : <http://www.w3.org/2001/02pd/rec54#>
         PREFIX dc: <http://purl.org/dc/elements/1.1/>
         PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
-        # PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
         PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
-        SELECT ?link ?title ?date ?version_of
-        WHERE {
-          ?link dc:title ?title ; dc:date ?date ; doc:versionOf ?version_of .
-        }
+        SELECT ?link ?title ?date
+        WHERE { ?link dc:title ?title ; dc:date ?date . }
       ))
-      data.query sse
+      rdf.query sse
     end
 
     #
@@ -92,13 +157,16 @@ module RelatonW3c
     #
     # @return [Array<RDF::Query::Solution>] query results
     #
-    def query_unversioned_docs
+    def query_unversioned_docs(rdf)
       sse = SPARQL.parse(%(
         PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
         SELECT ?version_of
-        WHERE { ?x doc:versionOf ?version_of . }
+        WHERE {
+          ?link doc:versionOf ?version_of .
+          FILTER ( isURI(?link) && isURI(?version_of) && ?link != ?version_of )
+        }
       ))
-      data.query(sse).uniq &:version_of
+      rdf.query(sse).uniq { |s| s.version_of.to_s.sub(/^https?:\/\//, "").sub(/\/$/, "") }
     end
 
     #

data/lib/relaton_w3c/data_index.rb

@@ -14,15 +14,24 @@ module RelatonW3c
     end
 
     #
-    # Add document to index
+    # Add document to index or update it if already exists
     #
     # @param [String] docnumber document number
     # @param [String] file path to document file
     #
-    def add(docnumber, file)
+    def add(docnumber, file) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
       dnparts = self.class.docnumber_to_parts docnumber
-      dnparts[:file] = file
-      @index << dnparts
+      rec = @index.detect { |i| i[:file] == file }
+      if rec
+        rec[:code] = dnparts[:code]
+        dnparts[:stage] ? rec[:stage] = dnparts[:stage] : rec.delete(:stage)
+        dnparts[:type] ? rec[:type] = dnparts[:type] : rec.delete(:type)
+        dnparts[:date] ? rec[:date] = dnparts[:date] : rec.delete(:date)
+        dnparts[:suff] ? rec[:suff] = dnparts[:suff] : rec.delete(:suff)
+      else
+        dnparts[:file] = file
+        @index << dnparts
+      end
     end
 
     #
@@ -111,18 +120,27 @@ module RelatonW3c
       #
       # @return [RelatonW3c::DataIndex] data index
       #
-      def create_from_repo # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-        resp = Zip::InputStream.new URI("#{W3cBibliography::SOURCE}index-w3c.zip").open
+      def create_from_repo
+        uri = URI("#{W3cBibliography::SOURCE}index-w3c.zip").open
+        resp = Zip::InputStream.new uri
         zip = resp.get_next_entry
+        index = RelatonBib.parse_yaml(zip.get_input_stream.read, [Symbol])
+        new index: index
+      end
 
-        # Newer versions of Psych uses the `permitted_classes:` parameter
-        index = if YAML.method(:safe_load).parameters.collect(&:last).index(:permitted_classes)
-                  YAML.safe_load(zip.get_input_stream.read, permitted_classes: [Symbol])
-                else
-                  YAML.safe_load(zip.get_input_stream.read, [Symbol])
+      #
+      # Create index from a file
+      #
+      # @param [String] index_file path to index file
+      #
+      # @return [RelatonW3c::DataIndex] data index
+      #
+      def create_from_file(index_file = "index-w3c.yaml")
+        index = if File.exist?(index_file)
+                  RelatonBib.parse_yaml(File.read(index_file), [Symbol])
+                else []
                 end
-
-        DataIndex.new index: index
+        new index_file: index_file, index: index
       end
 
       #

data/lib/relaton_w3c/data_parser.rb

@@ -25,7 +25,8 @@ module RelatonW3c
     # @param [RDF::Query::Solution] sol entry from the SPARQL query
     # @param [RelatonW3c::DataFetcher] fetcher data fetcher
     #
-    def initialize(sol, fetcher)
+    def initialize(rdf, sol, fetcher)
+      @rdf = rdf
       @sol = sol
       @fetcher = fetcher
     end
@@ -38,8 +39,8 @@ module RelatonW3c
     #
     # @return [RelatonW3c:W3cBibliographicItem, nil] bibliographic item
     #
-    def self.parse(sol, fetcher)
-      new(sol, fetcher).parse
+    def self.parse(rdf, sol, fetcher)
+      new(rdf, sol, fetcher).parse
     end
 
     #
@@ -100,7 +101,7 @@ module RelatonW3c
     #
     def parse_link
       link = @sol.respond_to?(:link) ? @sol.link : @sol.version_of
-      [RelatonBib::TypedUri.new(type: "src", content: link.to_s)]
+      [RelatonBib::TypedUri.new(type: "src", content: link.to_s.strip)]
     end
 
     #
@@ -131,7 +132,7 @@ module RelatonW3c
     #
     def identifier(link = nil)
       url = link || (@sol.respond_to?(:link) ? @sol.link : @sol.version_of)
-      self.class.parse_identifier(url.to_s)
+      self.class.parse_identifier(url.to_s.strip)
     end
 
     #
@@ -142,7 +143,7 @@ module RelatonW3c
     # @return [String] identifier
     #
     def self.parse_identifier(url)
-      if /.+\/(\w+(?:-[\w.]+)+(?:\/\w+)?)/ =~ url.to_s
+      if /.+\/(\w+(?:[-+][\w.]+)+(?:\/\w+)?)/ =~ url.to_s
         $1.to_s
       else url.to_s.split("/").last
       end
@@ -184,10 +185,10 @@ module RelatonW3c
           PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
           SELECT ?type
           WHERE {
-            { <#{@sol.link}> rdf:type ?type }
+            { <#{@sol.link.to_s.strip}> rdf:type ?type }
           }
         ))
-        @fetcher.data.query(sse).map { |s| s.type.to_s.split("#").last }
+        @rdf.query(sse).map { |s| s.type.to_s.split("#").last }
       end
     end
 
@@ -220,7 +221,7 @@ module RelatonW3c
       if @sol.respond_to?(:link)
         relations + editor_drafts
       else
-        document_versions.map { |r| create_relation(r.link.to_s, "hasEdition") }
+        document_versions.map { |r| create_relation(r.link.to_s.strip, "hasEdition") }
       end
     end
 
@@ -253,9 +254,9 @@ module RelatonW3c
         PREFIX : <http://www.w3.org/2001/02pd/rec54#>
         PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
         SELECT ?rel
-        WHERE { <#{@sol.link}> :ED ?rel . }
+        WHERE { <#{@sol.link.to_s.strip}> :ED ?rel . }
       ))
-      @fetcher.data.query(sse).map do |s|
+      @rdf.query(sse).map do |s|
         create_relation(s.rel.to_s, "hasDraft", "Editor's draft")
       end
     end
@@ -273,30 +274,53 @@ module RelatonW3c
         PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
         PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
         SELECT ?rel
-        WHERE { <#{@sol.link}> #{predicate} ?rel . }
+        WHERE { <#{@sol.link.to_s.strip}> #{predicate} ?rel . }
       ))
-      @fetcher.data.query(sse).order_by(:rel)
+      @rdf.query(sse).order_by(:rel)
     end
 
     #
     # Query document versions relations
     #
-    # @return [RDF::Query::Solutions] query results
+    # @return [Array<RDF::Query::Solution>] query results
     #
     def document_versions # rubocop:disable Metrics/MethodLength
-      @document_versions ||= begin
+      @document_versions ||= version_of.each_with_object([]) do |s, acc|
         sse = SPARQL.parse(%(
           PREFIX : <http://www.w3.org/2001/02pd/rec54#>
           PREFIX dc: <http://purl.org/dc/elements/1.1/>
           PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
           PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
           SELECT ?link ?title ?date
-          WHERE { ?link doc:versionOf <#{@sol.version_of}> ; dc:title ?title ; dc:date ?date }
+          WHERE {
+            ?link doc:versionOf <#{s.version_of}> ;
+            dc:title ?title ;
+            dc:date ?date .
+          }
         ))
-        @fetcher.data.query(sse)
+        @rdf.query(sse).each { |r| acc << r }
       end
     end
 
+    #
+    # Query for document versions
+    #
+    # @return [RDF::Query::Solutions] query results
+    #
+    def version_of
+      return [@sol] unless @sol.respond_to?(:link)
+
+      sse = SPARQL.parse(%(
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        SELECT ?version_of
+        WHERE {
+          <#{@sol.link.to_s.strip}> doc:versionOf ?version_of .
+          FILTER ( isURI(?version_of) && <#{@sol.link.to_s.strip}> != str(?version_of) )
+        }
+      ))
+      @rdf.query(sse)
+    end
+
     #
     # Create relation
     #
@@ -339,10 +363,10 @@ module RelatonW3c
         PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>
         SELECT ?full_name
         WHERE {
-          <#{@sol.link}> :editor/contact:fullName ?full_name
+          <#{@sol.link.to_s.strip}> :editor/contact:fullName ?full_name
         }
       ))
-      @fetcher.data.query(sse).order_by(:full_name).map do |ed|
+      @rdf.query(sse).order_by(:full_name).map do |ed|
         cn = RelatonBib::LocalizedString.new(ed.full_name.to_s, "en", "Latn")
         n = RelatonBib::FullName.new completename: cn
         p = RelatonBib::Person.new name: n
@@ -363,12 +387,13 @@ module RelatonW3c
         PREFIX contact: <http://www.w3.org/2000/10/swap/pim/contact#>
         SELECT ?home_page
         WHERE {
-          <#{@sol.link}> org:deliveredBy/contact:homePage ?home_page
+          <#{@sol.link.to_s.strip}> org:deliveredBy/contact:homePage ?home_page
         }
       ))
-      res = @fetcher.data.query(sse).order_by(:home_page)
+      res = @rdf.query(sse).order_by(:home_page)
       tc = res.each_with_object([]) do |edg, obj|
-        wg = @fetcher.group_names[edg.home_page.to_s.sub(/\/$/, "")]
+        group_path = edg.home_page.to_s.sub(/^https?:\/\//, "").sub(/\/$/, "")
+        wg = @fetcher.group_names[group_path]
         if wg
           rwg = RelatonBib::WorkGroup.new name: wg["name"]
           obj << RelatonBib::TechnicalCommittee.new(rwg)

data/lib/relaton_w3c/processor.rb

@@ -9,7 +9,7 @@ module RelatonW3c
       @prefix = "W3C"
       @defaultprefix = %r{^W3C\s}
       @idtype = "W3C"
-      @datasets = %w[w3c-rdf]
+      @datasets = %w[w3c-rdf w3c-tr-archive]
     end
 
     # @param code [String]
@@ -28,8 +28,8 @@ module RelatonW3c
     # @option opts [String] :output directory to output documents
     # @option opts [String] :format
     #
-    def fetch_data(_source, opts)
-      DataFetcher.fetch(**opts)
+    def fetch_data(source, opts)
+      DataFetcher.fetch(source, **opts)
     end
 
     # @param xml [String]

data/lib/relaton_w3c/version.rb

@@ -1,3 +1,3 @@
 module RelatonW3c
-  VERSION = "1.12.0".freeze
+  VERSION = "1.12.3".freeze
 end