relaton-w3c 1.10.1 → 1.11.2

data/grammars/isodoc.rng

@@ -152,9 +152,7 @@
             <data type="boolean"/>
           </attribute>
         </optional>
-        <oneOrMore>
-          <ref name="PureTextElement"/>
-        </oneOrMore>
+        <ref name="XrefBody"/>
       </element>
     </define>
     <define name="erefType">
@@ -188,6 +186,42 @@
         <ref name="PureTextElement"/>
       </oneOrMore>
     </define>
+    <define name="localityStack">
+      <element name="localityStack">
+        <optional>
+          <attribute name="connective">
+            <choice>
+              <value>and</value>
+              <value>or</value>
+              <value>from</value>
+              <value>to</value>
+              <value/>
+            </choice>
+          </attribute>
+        </optional>
+        <zeroOrMore>
+          <ref name="locality"/>
+        </zeroOrMore>
+      </element>
+    </define>
+    <define name="sourceLocalityStack">
+      <element name="sourceLocalityStack">
+        <optional>
+          <attribute name="connective">
+            <choice>
+              <value>and</value>
+              <value>or</value>
+              <value>from</value>
+              <value>to</value>
+              <value/>
+            </choice>
+          </attribute>
+        </optional>
+        <zeroOrMore>
+          <ref name="sourceLocality"/>
+        </zeroOrMore>
+      </element>
+    </define>
     <define name="ul">
       <element name="ul">
         <attribute name="id">
@@ -1098,6 +1132,16 @@
     </define>
   </include>
   <!-- end overrides -->
+  <define name="image" combine="choice">
+    <element name="svg">
+      <oneOrMore>
+        <choice>
+          <text/>
+          <ref name="AnyElement"/>
+        </choice>
+      </oneOrMore>
+    </element>
+  </define>
   <define name="MultilingualRenderingType">
     <choice>
       <value>common</value>
@@ -2631,4 +2675,30 @@
       </zeroOrMore>
     </element>
   </define>
+  <define name="XrefBody">
+    <zeroOrMore>
+      <ref name="XrefTarget"/>
+    </zeroOrMore>
+    <oneOrMore>
+      <ref name="PureTextElement"/>
+    </oneOrMore>
+  </define>
+  <define name="XrefTarget">
+    <element name="location">
+      <attribute name="target">
+        <data type="string">
+          <param name="pattern">\i\c*|\c+#\c+</param>
+        </data>
+      </attribute>
+      <attribute name="connective">
+        <choice>
+          <value>and</value>
+          <value>or</value>
+          <value>from</value>
+          <value>to</value>
+          <value/>
+        </choice>
+      </attribute>
+    </element>
+  </define>
 </grammar>

data/lib/relaton_w3c/bibxml_parser.rb

@@ -13,5 +13,12 @@ module RelatonW3c
     def pubid_type(_)
       "W3C"
     end
+
+    def docids(reference, ver)
+      ids = super
+      ids.reject! &:primary
+      id = "W3C #{reference[:target].split('/').last}"
+      ids.unshift RelatonBib::DocumentIdentifier.new(id: id, type: "W3C", primary: true)
+    end
   end
 end

data/lib/relaton_w3c/data_fetcher.rb

@@ -0,0 +1,188 @@
+require "rdf"
+require "linkeddata"
+require "sparql"
+require "mechanize"
+require "relaton_w3c/data_parser"
+
+module RelatonW3c
+  class DataFetcher
+    attr_reader :data, :group_names
+
+    #
+    # Data fetcher initializer
+    #
+    # @param [String] output directory to save files
+    # @param [String] format format of output files (xml, yaml, bibxml)
+    #
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+      dir = File.dirname(File.expand_path(__FILE__))
+      @group_names = YAML.load_file(File.join(dir, "workgroups.yaml"))
+      @data = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
+      @files = []
+      @index = DataIndex.new
+    end
+
+    #
+    # Initialize fetcher and run fetch
+    #
+    # @param [Strin] output directory to save files, default: "data"
+    # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
+    #
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Parse documents
+    #
+    def fetch # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      query_versioned_docs.each do |sl|
+        save_doc DataParser.parse(sl, self)
+      rescue StandardError => e
+        warn "Error: document #{sl.link} #{e.message}"
+        warn e.backtrace.join("\n")
+      end
+      query_unversioned_docs.each do |sl|
+        save_doc DataParser.parse(sl, self)
+      rescue StandardError => e
+        warn "Error: document #{sl.version_of} #{e.message}"
+        warn e.backtrace.join("\n")
+      end
+      Dir[File.expand_path("../../data/*", __dir__)].each do |file|
+        xml = File.read file, encoding: "UTF-8"
+        save_doc BibXMLParser.parse(xml), warn_duplicate: false
+      rescue StandardError => e
+        warn "Error: document #{file} #{e.message}"
+        warn e.backtrace.join("\n")
+      end
+      @index.sort!.save
+    end
+
+    #
+    # Create index file
+    #
+    # def create_index
+    #   index_file = "index-w3c.yaml"
+    #   index_yaml = @index.sort do |a, b|
+    #     compare_index_items a, b
+    #   end.to_yaml
+    #   File.write index_file, index_yaml, encoding: "UTF-8"
+    # end
+
+    #
+    # Compare index items
+    #
+    # @param [Hash] aid first item
+    # @param [Hash] bid second item
+    #
+    # @return [Integer] comparison result
+    #
+    # def compare_index_items(aid, bid) # rubocop:disable Metrics/AbcSize
+    #   ret = aid[:code] <=> bid[:code]
+    #   ret = stage_weight(bid[:stage]) <=> stage_weight(aid[:stage]) if ret.zero?
+    #   ret = date_weight(bid[:date]) <=> date_weight(aid[:date]) if ret.zero?
+    #   # ret = aid[:type] <=> bid[:type] if ret.zero?
+    #   ret
+    # end
+
+    #
+    # Weight of stage
+    #
+    # @param [String, nil] stage stage
+    #
+    # @return [Integer] weight
+    #
+    # def stage_weight(stage)
+    #   return DataParser::STAGES.size if stage.nil?
+
+    #   DataParser::STAGES.keys.index(stage)
+    # end
+
+    #
+    # Weight of date
+    #
+    # @param [String] date date
+    #
+    # @return [String] weight
+    #
+    # def date_weight(date)
+    #   return "99999999" if date.nil?
+
+    #   date
+    # end
+
+    #
+    # Query RDF source for documents
+    #
+    # @return [RDF::Query::Solutions] query results
+    #
+    def query_versioned_docs # rubocop:disable Metrics/MethodLength
+      sse = SPARQL.parse(%(
+        PREFIX : <http://www.w3.org/2001/02pd/rec54#>
+        PREFIX dc: <http://purl.org/dc/elements/1.1/>
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        # PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
+        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+        SELECT ?link ?title ?date ?version_of
+        WHERE {
+          ?link dc:title ?title ; dc:date ?date ; doc:versionOf ?version_of .
+        }
+      ))
+      data.query sse
+    end
+
+    def query_unversioned_docs
+      sse = SPARQL.parse(%(
+        PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
+        SELECT ?version_of
+        WHERE { ?x doc:versionOf ?version_of . }
+      ))
+      data.query(sse).uniq &:version_of
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonW3c::W3cBibliographicItem, nil] bib bibliographic item
+    #
+    def save_doc(bib, warn_duplicate: true) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      return unless bib
+
+      c = case @format
+          when "xml" then bib.to_xml(bibdata: true)
+          when "yaml" then bib.to_hash.to_yaml
+          else bib.send("to_#{@format}")
+          end
+      # id = bib.docidentifier.detect(&:primary)&.id || bib.formattedref.content
+      file = file_name(bib.docnumber)
+      if @files.include?(file)
+        warn "File #{file} already exists. Document: #{bib.docnumber}" if warn_duplicate
+      else
+        @index.add bib.docnumber, file
+        @files << file
+        File.write file, c, encoding: "UTF-8"
+      end
+    end
+
+    #
+    # Generate file name
+    #
+    # @param [String] id document id
+    #
+    # @return [String] file name
+    #
+    def file_name(id)
+      name = id.sub(/^W3C\s/, "").gsub(/[\s,:\/+]/, "_").squeeze("_").downcase
+      File.join @output, "#{name}.#{@ext}"
+    end
+  end
+end

data/lib/relaton_w3c/data_index.rb

@@ -0,0 +1,143 @@
+module RelatonW3c
+  class DataIndex
+    #
+    # Initialize data index.
+    #
+    # @param [String] index_file path to index file
+    # @param [Array<Hash>] index index data
+    #
+    def initialize(index_file: "index-w3c.yaml", index: [])
+      @index_file = index_file
+      @index = index
+    end
+
+    #
+    # Create index from a GitHub repository
+    #
+    # @return [RelatonW3c::DataIndex] data index
+    #
+    def self.create_from_repo
+      resp_index = Net::HTTP.get(URI("#{W3cBibliography::SOURCE}index-w3c.yaml"))
+
+      # Newer versions of Psych uses the `permitted_classes:` parameter
+      index = if YAML.method(:safe_load).parameters.collect(&:last).index(:permitted_classes)
+        YAML.safe_load(resp_index, permitted_classes: [Symbol])
+      else
+        YAML.safe_load(resp_index, [Symbol])
+      end
+
+      DataIndex.new index: index
+    end
+
+    #
+    # Add document to index
+    #
+    # @param [String] docnumber document number
+    # @param [String] file path to document file
+    #
+    def add(docnumber, file)
+      @index << docnumber_to_parts(docnumber, file)
+    end
+
+    #
+    # Save index to file.
+    #
+    def save
+      File.write @index_file, @index.to_yaml, encoding: "UTF-8"
+    end
+
+    #
+    # Sort index
+    #
+    # @return [Array<Hash>] sorted index
+    #
+    def sort!
+      @index.sort! { |a, b| compare_index_items a, b }
+      self
+    end
+
+    #
+    # Search filename in index
+    #
+    # @param [String] ref reference
+    #
+    # @return [String] document's filename
+    #
+    def search(ref) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      dparts = docnumber_to_parts(ref)
+      @index.detect do |parts|
+        parts[:code].match?(/^#{Regexp.escape dparts[:code]}/i) &&
+          (dparts[:stage].nil? || dparts[:stage].casecmp?(parts[:stage])) &&
+          (dparts[:type].nil? || dparts[:type].casecmp?(parts[:type])) &&
+          (dparts[:date].nil? || dparts[:date] == parts[:date]) &&
+          (dparts[:suff].nil? || dparts[:suff].casecmp?(parts[:suff]))
+      end&.fetch(:file)
+    end
+
+    #
+    # Compare index items
+    #
+    # @param [Hash] aid first item
+    # @param [Hash] bid second item
+    #
+    # @return [Integer] comparison result
+    #
+    def compare_index_items(aid, bid) # rubocop:disable Metrics/AbcSize
+      ret = aid[:code] <=> bid[:code]
+      ret = stage_weight(bid[:stage]) <=> stage_weight(aid[:stage]) if ret.zero?
+      ret = date_weight(bid[:date]) <=> date_weight(aid[:date]) if ret.zero?
+      # ret = aid[:type] <=> bid[:type] if ret.zero?
+      ret
+    end
+
+    #
+    # Weight of stage
+    #
+    # @param [String, nil] stage stage
+    #
+    # @return [Integer] weight
+    #
+    def stage_weight(stage)
+      return DataParser::STAGES.size if stage.nil?
+
+      DataParser::STAGES.keys.index(stage)
+    end
+
+    #
+    # Weight of date
+    #
+    # @param [String] date date
+    #
+    # @return [String] weight
+    #
+    def date_weight(date)
+      return "99999999" if date.nil?
+
+      date
+    end
+
+    #
+    # Parse document number to parts
+    #
+    # @param [String] docnumber document number
+    # @param [String, nil] file path to document file
+    #
+    # @return [Hash{Symbol=>String}] document parts
+    #
+    def docnumber_to_parts(docnumber, file = nil) # rubocop:disable Metrics/MethodLength
+      %r{
+        ^(?:(?:(?<stage>WD|CRD|CR|PR|PER|REC|SPSD|OBSL|RET)|(?<type>D?NOTE))-)?
+        (?<code>\w+(?:[+-][\w.]+)*?)
+        (?:-(?<date>\d{8}|\d{6}))?
+        (?:/(?<suff>\w+))?$
+      }xi =~ docnumber
+      entry = { code: code }
+      entry[:file] = file if file
+      entry[:stage] = stage if stage
+      entry[:type] = type if type
+      entry[:date] = date if date
+      entry[:suff] = suff if suff
+      entry
+    end
+  end
+end