relaton-w3c 1.19.0 → 1.20.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dde70fdc615a88c4343add641e193635a5377af68180d96122b02af747aa50e5
-  data.tar.gz: 363d74d0a170b307006310bbbe2f6e645da40aabf12dbc2a94ebac4887f2452f
+  metadata.gz: beb3bb1a8550059e754f001412336b78ab55771f25ebf33534dd92f59a724b9d
+  data.tar.gz: d32d55f604c16613c04d5d2ad5554e7c7ed8c6151ad0218411c47d58099b5a63
 SHA512:
-  metadata.gz: fec1491eaa3108fd8726dfa0af50f54b98d7c28286c936d827088eed8ee7badbe6be430dd4588ad6324936f6cce8824379750a71ac4da9aa8d88788a47fceec4
-  data.tar.gz: e9d4074458f1a01e12ab84619861dda851ce668573d3b3e843a2c7ba8effed42e6c94545b49157626ba922ebfc31b57ff9d5877461626d2578a9ebdf128d314c
+  metadata.gz: b2927bebbb35dde93cb2659b44e0ef47608ffa164ae33d138041a46139ca17b1a62061eb1055959fe91771bf397bf5ecb11f9ff2ca19055d7d804a65962a8974
+  data.tar.gz: 240f1c9f84770eb129a372a015b582c7f3ff6870eaf83e2dd723ff9538bbc19da34d58190234441b3c6b9dde224810e09c5e287473afa1183f23c53deabcc849

data/README.adoc

@@ -108,16 +108,13 @@ RelatonW3c::W3cBibliographicItem.new **bib_hash
 
 === Fetch data
 
-The method `RelatonW3c::DataFetcher.fetch(source, output: "data", format: "yaml")` converts all the documents from the dataset and saves them to the `./data` folder in YAML format.
+The method `RelatonW3c::DataFetcher.fetch(output: "data", format: "yaml")` converts all the documents from the dataset and saves them to the `./data` folder in YAML format.
 Arguments:
 
-- `source` - the name of the dataset (`w3c-rdf` or `w3c-tr-archive`)
 - `output` - folder to save documents (default './data').
 - `format` - the format in which the documents are saved. Possible formats are: `yaml`, `xml`, `bibxml` (default `yaml`).
 
-The available datasets are:
-- `w3c-rdf` - The dataset is fetched from http://www.w3.org/2002/01/tr-automation/tr.rdf.
-- `w3c-tr-archive` - The archive dataset files should be downloaded from https://github.com/relaton/w3c-tr-archive repository and placed into `w3c-tr-archive` folder.
+The method uses https://api.w3.org/doc API to fetch all the W3C documents.
 
 [source,ruby]
 ----

data/lib/relaton_w3c/data_fetcher.rb

@@ -1,12 +1,10 @@
-require "rdf"
-require "linkeddata"
-require "sparql"
-require "mechanize"
-require "relaton_w3c/data_parser"
+require "w3c_api"
+require_relative "rate_limit_handler"
+require_relative "data_parser"
 
 module RelatonW3c
   class DataFetcher
-    attr_reader :data, :group_names
+    include RelatonW3c::RateLimitHandler
 
     #
     # Data fetcher initializer
@@ -18,8 +16,8 @@ module RelatonW3c
       @output = output
       @format = format
       @ext = format.sub(/^bib/, "")
-      dir = File.dirname(File.expand_path(__FILE__))
-      @group_names = YAML.load_file(File.join(dir, "workgroups.yaml"))
+      @files = Set.new
+      @fetched_urls = {}
       @index = DataIndex.create_from_file
       @index1 = Relaton::Index.find_or_create :W3C, file: "index1.yaml"
     end
@@ -27,178 +25,59 @@ module RelatonW3c
     #
     # Initialize fetcher and run fetch
     #
-    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
     # @param [Strin] output directory to save files, default: "data"
     # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
     #
-    def self.fetch(source, output: "data", format: "yaml")
+    def self.fetch(output: "data", format: "yaml")
       t1 = Time.now
       puts "Started at: #{t1}"
       FileUtils.mkdir_p output
-      new(output, format).fetch source
+      new(output, format).fetch
       t2 = Time.now
       puts "Stopped at: #{t2}"
       puts "Done in: #{(t2 - t1).round} sec."
     end
 
-    #
-    # Parse documents
-    #
-    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
-    #
-    def fetch(source) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      each_dataset(source) do |rdf|
-        %i[versioned unversioned].each do |type|
-          send("query_#{type}_docs", rdf).each do |sl|
-            bib = DataParser.parse(rdf, sl, self)
-            add_has_edition_relation(bib) if type == :unversioned
-            save_doc bib
-          rescue StandardError => e
-            link = sl.respond_to?(:link) ? sl.link : sl.version_of
-            Util.error "Error: document #{link} #{e.message}\n#{e.backtrace.join("\n")}"
-          end
-        end
-      end
-      @index.sort!.save
-      @index1.save
+    def client
+      @client ||= W3cApi::Client.new
     end
 
     #
-    # Add hasEdition relations form previous parsed document
-    #
-    # @param [RelatonW3c::W3cBibliographicItem] bib bibligraphic item
+    # Parse documents
     #
-    def add_has_edition_relation(bib) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
-      file = file_name bib.docnumber
-      if File.exist? file
-        item = send "read_#{@format}", file
-        item.relation.each do |r1|
-          r1.type = "hasEdition" if r1.type == "instanceOf"
-          same_edition = bib.relation.detect { |r2| same_edition?(r1, r2) }
-          bib.relation << r1 unless same_edition
+    def fetch
+      specs = client.specifications
+      loop do
+        specs.links.specifications.each do |spec|
+          fetch_spec spec
         end
-      end
-      bib.relation.select { |r| r.type == "hasEdition" }
-        .max_by { |r| r.bibitem.id.match(/(?<=-)\d{8}$/).to_s }&.type = "instanceOf"
-    end
 
-    #
-    # Read XML file
-    #
-    # @param [String] file file name
-    #
-    # @return [RelatonW3c::W3cBibliographicItem] bibliographic item
-    #
-    def read_xml(file)
-      XMLParser.from_xml(File.read(file, encoding: "UTF-8"))
-    end
-
-    #
-    # Read YAML file
-    #
-    # @param [String] file file name
-    #
-    # @return [RelatonW3c::W3cBibliographicItem] bibliographic item
-    #
-    def read_yaml(file)
-      hash = YAML.load_file(file)
-      W3cBibliographicItem.from_hash(hash)
-    end
+        break unless specs.next?
 
-    #
-    # Read BibXML file
-    #
-    # @param [String] file file name
-    #
-    # @return [RelatonW3c::W3cBibliographicItem] bibliographic item
-    #
-    def read_bibxml(file)
-      BibXMLParser.parse File.read(file, encoding: "UTF-8")
+        specs = specs.next
+      end
+      @index.sort!.save
+      @index1.save
     end
 
-    #
-    # Compare two relations
-    #
-    # @param [RelatonW3c::W3cBibliographicItem] rel1 relation 1
-    # @param [RelatonW3c::W3cBibliographicItem] rel2 relation 2
-    #
-    # @return [Boolean] true if relations are same
-    #
-    def same_edition?(rel1, rel2)
-      return false unless rel1.type == "hasEdition" && rel1.type == rel2.type
-
-      ids1 = rel1.bibitem.docidentifier.map(&:id)
-      ids2 = rel2.bibitem.docidentifier.map(&:id)
-      (ids1 & ids2).any?
-    end
+    def fetch_spec(unrealized_spec)
+      spec = realize unrealized_spec
+      save_doc DataParser.parse(spec)
 
-    #
-    # Yield fetching for each dataset
-    #
-    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
-    #
-    # @yield [RDF::Repository] RDF repository
-    #
-    def each_dataset(source, &_block) # rubocop:disable Metrics/MethodLength
-      case source
-      when "w3c-tr-archive"
-        Dir["w3c-tr-archive/*.rdf"].map do |f|
-          @files = []
-          yield RDF::Repository.load(f)
-        end
-      when "w3c-rdf"
-        @files = []
-        rdf = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
-        yield rdf
-        # parse_static_dataset
+      if spec.links.respond_to?(:version_history) && spec.links.version_history
+        version_history = realize spec.links.version_history
+        version_history.links.spec_versions.each { |version| save_doc DataParser.parse(realize version) }
       end
-    end
-
-    #
-    # Parse static dataset
-    #
-    # def parse_static_dataset
-    #   Dir[File.expand_path("../../data/*", __dir__)].each do |file|
-    #     xml = File.read file, encoding: "UTF-8"
-    #     save_doc BibXMLParser.parse(xml), warn_duplicate: false
-    #   rescue StandardError => e
-    #     warn "Error: document #{file} #{e.message}"
-    #     warn e.backtrace.join("\n")
-    #   end
-    # end
 
-    #
-    # Query RDF source for versioned documents
-    #
-    # @return [RDF::Query::Solutions] query results
-    #
-    def query_versioned_docs(rdf)
-      sse = SPARQL.parse(%(
-        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
-        PREFIX dc: <http://purl.org/dc/elements/1.1/>
-        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
-        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
-        SELECT ?link ?title ?date
-        WHERE { ?link dc:title ?title ; dc:date ?date . }
-      ))
-      rdf.query sse
-    end
+      if spec.links.respond_to?(:predecessor_versions) && spec.links.predecessor_versions
+        predecessor_versions = realize spec.links.predecessor_versions
+        predecessor_versions.links.predecessor_versions.each { |version| save_doc DataParser.parse(realize version) }
+      end
 
-    #
-    # Query RDF source for unversioned documents
-    #
-    # @return [Array<RDF::Query::Solution>] query results
-    #
-    def query_unversioned_docs(rdf)
-      sse = SPARQL.parse(%(
-        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
-        SELECT ?version_of
-        WHERE {
-          ?link doc:versionOf ?version_of .
-          FILTER ( isURI(?link) && isURI(?version_of) && ?link != ?version_of )
-        }
-      ))
-      rdf.query(sse).uniq { |s| s.version_of.to_s.sub(/^https?:\/\//, "").sub(/\/$/, "") }
+      if spec.links.respond_to?(:successor_versions) && spec.links.successor_versions
+        successor_versions = realize spec.links.successor_versions
+        successor_versions.links.successor_versions.each { |version| save_doc DataParser.parse(realize version) }
+      end
     end
 
     #
@@ -206,15 +85,9 @@ module RelatonW3c
     #
     # @param [RelatonW3c::W3cBibliographicItem, nil] bib bibliographic item
     #
-    def save_doc(bib, warn_duplicate: true) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    def save_doc(bib, warn_duplicate: true)
       return unless bib
 
-      c = case @format
-          when "xml" then bib.to_xml(bibdata: true)
-          when "yaml" then bib.to_hash.to_yaml
-          else bib.send("to_#{@format}")
-          end
-      # id = bib.docidentifier.detect(&:primary)&.id || bib.formattedref.content
       file = file_name(bib.docnumber)
       if @files.include?(file)
         Util.warn "File #{file} already exists. Document: #{bib.docnumber}" if warn_duplicate
@@ -223,7 +96,15 @@ module RelatonW3c
         @index.add pubid, file
         @index1.add_or_update pubid.to_hash, file
         @files << file
-        File.write file, c, encoding: "UTF-8"
+      end
+      File.write file, serialize(bib), encoding: "UTF-8"
+    end
+
+    def serialize(bib)
+      case @format
+      when "xml" then bib.to_xml(bibdata: true)
+      when "yaml" then bib.to_hash.to_yaml
+      else bib.send("to_#{@format}")
       end
     end