relaton-w3c 1.11.6 → 1.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: faf78f2abf348e9aa6dd74f2f0fe94899d9d6fc34af3368849a818b6dd856e26
-  data.tar.gz: c4cabbd9559b9f122f34fbb204f7780606bfc87c9db999492fbdd2b6b02639b3
+  metadata.gz: 0bf7e3fdd349e94b272db7045c88bae46fe32f642436527486ff4db8abd08518
+  data.tar.gz: d66405c0f27950d42f1ceac65d3be95a7f93e19ff5045d4c4618ddb551d819b1
 SHA512:
-  metadata.gz: cc6e6c80914b8973091795b1f234ea57a49499bb0466ae04a14358c30e526c38a6f607031e9dd0d4963c2caccca26dd9418939c871b4f47016aa094156d783a3
-  data.tar.gz: 041625ce66ae2949112bc20856de89b05fd4772c1dd1f3413a3a5b04059ec7cb84f8b92517a3634294080031760fca8795b7ae113bbfae03986f775c5f52acb0
+  metadata.gz: fb14cc520b1fbe3f5bc0dfd387b990b1693e5e7ab8c15d093372cf8ebe5f11ae4df521c5d05885da83ef60b3fe475dedaeb3dc764a9aaad22f9288de50307ede
+  data.tar.gz: 5d36eea643697240c15bc24abcc6b03414731fd1aed4dc48f4e86f32c7feba6cfa99e3e6f1503882b241414875534a480f4e3c215a39b9bccb60bf0d3c798df5

data/README.adoc

@@ -151,14 +151,17 @@ RelatonW3c::W3cBibliographicItem.new **bib_hash
 
 === Fetch data
 
-There is a W3C dataset http://www.w3.org/2002/01/tr-automation/tr.rdf which can be converted into RelatonXML/BibXML/BibYAML formats. The static files from the `path_to_gem/withdrawn` directory are added to output during fetching data.
-
-The method `RelatonW3c::DataFetcher.fetch(output: "data", format: "yaml")` converts all the documents from the dataset and save them to the `./data` folder in YAML format.
+The method `RelatonW3c::DataFetcher.fetch(source, output: "data", format: "yaml")` converts all the documents from the dataset and save them to the `./data` folder in YAML format.
 Arguments:
 
+- `source` - name of dataset (`w3c-rdf` or `w3c-tr-archive`)
 - `output` - folder to save documents (default './data').
 - `format` - format in which the documents are saved. Possible formats are: `yaml`, `xml`, `bibxml` (default `yaml`).
 
+The available datasets are:
+- `w3c-rdf` - The dataset is fetched from http://www.w3.org/2002/01/tr-automation/tr.rdf.
+- `w3c-tr-archive` - The archive dataset files should be downloaded from https://github.com/relaton/w3c-tr-archive repository and placed into `w3c-tr-archive` folder.
+
 [source,ruby]
 ----
 RelatonW3c::DataFetcher.fetch

data/grammars/biblio.rng

@@ -527,7 +527,7 @@
   </define>
   <define name="LocalityType">
     <data type="string">
-      <param name="pattern">section|clause|part|paragraph|chapter|page|whole|table|annex|figure|note|list|example|volume|issue|time|anchor|locality:[a-zA-Z0-9_]+</param>
+      <param name="pattern">section|clause|part|paragraph|chapter|page|title|line|whole|table|annex|figure|note|list|example|volume|issue|time|anchor|locality:[a-zA-Z0-9_]+</param>
     </data>
   </define>
   <define name="referenceFrom">
@@ -614,12 +614,103 @@
     <optional>
       <ref name="fetched"/>
     </optional>
-    <choice>
-      <oneOrMore>
-        <ref name="btitle"/>
-      </oneOrMore>
+    <optional>
       <ref name="formattedref"/>
-    </choice>
+    </optional>
+    <oneOrMore>
+      <ref name="btitle"/>
+    </oneOrMore>
+    <zeroOrMore>
+      <ref name="bsource"/>
+    </zeroOrMore>
+    <oneOrMore>
+      <ref name="docidentifier"/>
+    </oneOrMore>
+    <optional>
+      <ref name="docnumber"/>
+    </optional>
+    <zeroOrMore>
+      <ref name="bdate"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="contributor"/>
+    </zeroOrMore>
+    <optional>
+      <ref name="edition"/>
+    </optional>
+    <zeroOrMore>
+      <ref name="version"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="biblionote"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="language"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="script"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="bibabstract"/>
+    </zeroOrMore>
+    <optional>
+      <ref name="status"/>
+    </optional>
+    <zeroOrMore>
+      <ref name="copyright"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="docrelation"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="series"/>
+    </zeroOrMore>
+    <optional>
+      <ref name="medium"/>
+    </optional>
+    <zeroOrMore>
+      <ref name="bplace"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="bprice"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="extent"/>
+    </zeroOrMore>
+    <optional>
+      <ref name="bibliographic_size"/>
+    </optional>
+    <zeroOrMore>
+      <ref name="accesslocation"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="license"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="bclassification"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="bkeyword"/>
+    </zeroOrMore>
+    <optional>
+      <ref name="validity"/>
+    </optional>
+  </define>
+  <define name="ReducedBibliographicItem">
+    <optional>
+      <attribute name="type">
+        <ref name="BibItemType"/>
+      </attribute>
+    </optional>
+    <optional>
+      <ref name="fetched"/>
+    </optional>
+    <optional>
+      <ref name="formattedref"/>
+    </optional>
+    <zeroOrMore>
+      <ref name="btitle"/>
+    </zeroOrMore>
     <zeroOrMore>
       <ref name="bsource"/>
     </zeroOrMore>
@@ -638,9 +729,9 @@
     <optional>
       <ref name="edition"/>
     </optional>
-    <optional>
+    <zeroOrMore>
       <ref name="version"/>
-    </optional>
+    </zeroOrMore>
     <zeroOrMore>
       <ref name="biblionote"/>
     </zeroOrMore>
@@ -833,6 +924,12 @@
           <data type="boolean"/>
         </attribute>
       </optional>
+      <optional>
+        <attribute name="language"/>
+      </optional>
+      <optional>
+        <attribute name="script"/>
+      </optional>
       <text/>
     </element>
   </define>
@@ -851,17 +948,46 @@
   </define>
   <define name="bplace">
     <element name="place">
-      <optional>
-        <attribute name="uri">
-          <data type="anyURI"/>
-        </attribute>
-      </optional>
-      <optional>
-        <attribute name="region"/>
-      </optional>
+      <choice>
+        <text/>
+        <group>
+          <ref name="bibliocity"/>
+          <zeroOrMore>
+            <ref name="biblioregion"/>
+          </zeroOrMore>
+          <zeroOrMore>
+            <ref name="bibliocountry"/>
+          </zeroOrMore>
+        </group>
+      </choice>
+    </element>
+  </define>
+  <define name="bibliocity">
+    <element name="city">
       <text/>
     </element>
   </define>
+  <define name="biblioregion">
+    <element name="region">
+      <ref name="RegionType"/>
+    </element>
+  </define>
+  <define name="bibliocountry">
+    <element name="country">
+      <ref name="RegionType"/>
+    </element>
+  </define>
+  <define name="RegionType">
+    <optional>
+      <attribute name="iso"/>
+    </optional>
+    <optional>
+      <attribute name="recommended">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <text/>
+  </define>
   <define name="bprice">
     <element name="price">
       <attribute name="currency"/>
@@ -957,36 +1083,34 @@
           <ref name="SeriesType"/>
         </attribute>
       </optional>
-      <choice>
+      <optional>
         <ref name="formattedref"/>
-        <group>
-          <ref name="btitle"/>
-          <optional>
-            <ref name="bplace"/>
-          </optional>
-          <optional>
-            <ref name="seriesorganization"/>
-          </optional>
-          <optional>
-            <ref name="abbreviation"/>
-          </optional>
-          <optional>
-            <ref name="seriesfrom"/>
-          </optional>
-          <optional>
-            <ref name="seriesto"/>
-          </optional>
-          <optional>
-            <ref name="seriesnumber"/>
-          </optional>
-          <optional>
-            <ref name="seriespartnumber"/>
-          </optional>
-          <optional>
-            <ref name="seriesrun"/>
-          </optional>
-        </group>
-      </choice>
+      </optional>
+      <ref name="btitle"/>
+      <optional>
+        <ref name="bplace"/>
+      </optional>
+      <optional>
+        <ref name="seriesorganization"/>
+      </optional>
+      <optional>
+        <ref name="abbreviation"/>
+      </optional>
+      <optional>
+        <ref name="seriesfrom"/>
+      </optional>
+      <optional>
+        <ref name="seriesto"/>
+      </optional>
+      <optional>
+        <ref name="seriesnumber"/>
+      </optional>
+      <optional>
+        <ref name="seriespartnumber"/>
+      </optional>
+      <optional>
+        <ref name="seriesrun"/>
+      </optional>
     </element>
   </define>
   <define name="SeriesType">
@@ -1145,7 +1269,7 @@
         </element>
       </optional>
       <element name="bibitem">
-        <ref name="BibliographicItem"/>
+        <ref name="ReducedBibliographicItem"/>
       </element>
       <choice>
         <zeroOrMore>
@@ -1170,9 +1294,9 @@
       <optional>
         <ref name="revision-date"/>
       </optional>
-      <zeroOrMore>
+      <optional>
         <ref name="draft"/>
-      </zeroOrMore>
+      </optional>
     </element>
   </define>
   <define name="vedition">

data/grammars/isodoc.rng

@@ -32,6 +32,43 @@
         <ref name="DocumentType"/>
       </element>
     </define>
+    <define name="admonition">
+      <element name="admonition">
+        <attribute name="type">
+          <ref name="AdmonitionType"/>
+        </attribute>
+        <optional>
+          <attribute name="class"/>
+        </optional>
+        <attribute name="id">
+          <data type="ID"/>
+        </attribute>
+        <optional>
+          <attribute name="uri">
+            <data type="anyURI"/>
+          </attribute>
+        </optional>
+        <optional>
+          <attribute name="coverpage">
+            <data type="boolean"/>
+          </attribute>
+        </optional>
+        <optional>
+          <attribute name="notag">
+            <data type="boolean"/>
+          </attribute>
+        </optional>
+        <optional>
+          <ref name="tname"/>
+        </optional>
+        <zeroOrMore>
+          <ref name="paragraph-with-footnote"/>
+        </zeroOrMore>
+        <zeroOrMore>
+          <ref name="note"/>
+        </zeroOrMore>
+      </element>
+    </define>
     <define name="index">
       <element name="index">
         <optional>
@@ -79,6 +116,11 @@
             <data type="boolean"/>
           </attribute>
         </optional>
+        <optional>
+          <attribute name="suppress_identifier">
+            <data type="boolean"/>
+          </attribute>
+        </optional>
         <ref name="BibliographicItem"/>
       </element>
     </define>
@@ -245,6 +287,9 @@
             <ref name="MultilingualRenderingType"/>
           </attribute>
         </optional>
+        <optional>
+          <ref name="tname"/>
+        </optional>
         <oneOrMore>
           <ref name="ul_li"/>
         </oneOrMore>
@@ -287,6 +332,9 @@
             </choice>
           </attribute>
         </optional>
+        <optional>
+          <ref name="tname"/>
+        </optional>
         <oneOrMore>
           <ref name="li"/>
         </oneOrMore>
@@ -323,6 +371,9 @@
             <ref name="MultilingualRenderingType"/>
           </attribute>
         </optional>
+        <optional>
+          <ref name="tname"/>
+        </optional>
         <oneOrMore>
           <ref name="dt"/>
           <ref name="dd"/>
@@ -657,6 +708,9 @@
       <optional>
         <attribute name="tag"/>
       </optional>
+      <optional>
+        <attribute name="type"/>
+      </optional>
       <optional>
         <attribute name="multilingual-rendering">
           <ref name="MultilingualRenderingType"/>
@@ -692,6 +746,9 @@
         <optional>
           <attribute name="tag"/>
         </optional>
+        <optional>
+          <attribute name="type"/>
+        </optional>
         <optional>
           <attribute name="multilingual-rendering">
             <ref name="MultilingualRenderingType"/>
@@ -851,6 +908,16 @@
             <ref name="MultilingualRenderingType"/>
           </attribute>
         </optional>
+        <optional>
+          <attribute name="coverpage">
+            <data type="boolean"/>
+          </attribute>
+        </optional>
+        <optional>
+          <attribute name="notag">
+            <data type="boolean"/>
+          </attribute>
+        </optional>
         <oneOrMore>
           <choice>
             <ref name="paragraph"/>
@@ -1178,6 +1245,7 @@
       <ref name="concept"/>
       <ref name="add"/>
       <ref name="del"/>
+      <ref name="span"/>
     </choice>
   </define>
   <define name="add">
@@ -1204,6 +1272,14 @@
       </choice>
     </element>
   </define>
+  <define name="span">
+    <element name="span">
+      <attribute name="class"/>
+      <oneOrMore>
+        <ref name="TextElement"/>
+      </oneOrMore>
+    </element>
+  </define>
   <define name="concept">
     <element name="concept">
       <optional>
@@ -1994,6 +2070,7 @@
       <value>compare</value>
       <value>contrast</value>
       <value>see</value>
+      <value>seealso</value>
     </choice>
   </define>
   <define name="deprecates">

data/lib/relaton_w3c/data_fetcher.rb

@@ -20,22 +20,21 @@ module RelatonW3c
       @ext = format.sub(/^bib/, "")
       dir = File.dirname(File.expand_path(__FILE__))
       @group_names = YAML.load_file(File.join(dir, "workgroups.yaml"))
-      @data = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
-      @files = []
       @index = DataIndex.new
     end
 
     #
     # Initialize fetcher and run fetch
     #
+    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
     # @param [Strin] output directory to save files, default: "data"
     # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
     #
-    def self.fetch(output: "data", format: "yaml")
+    def self.fetch(source, output: "data", format: "yaml")
       t1 = Time.now
       puts "Started at: #{t1}"
-      FileUtils.mkdir_p output unless Dir.exist? output
-      new(output, format).fetch
+      FileUtils.mkdir_p output
+      new(output, format).fetch source
       t2 = Time.now
       puts "Stopped at: #{t2}"
       puts "Done in: #{(t2 - t1).round} sec."
@@ -44,47 +43,113 @@ module RelatonW3c
     #
     # Parse documents
     #
-    def fetch # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      query_versioned_docs.each do |sl|
-        save_doc DataParser.parse(sl, self)
-      rescue StandardError => e
-        warn "Error: document #{sl.link} #{e.message}"
-        warn e.backtrace.join("\n")
+    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
+    #
+    def fetch(source) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      each_dataset(source) do |rdf|
+        %i[versioned unversioned].each do |type|
+          send("query_#{type}_docs", rdf).each do |sl|
+            bib = DataParser.parse(rdf, sl, self)
+            add_has_edition_relation(bib) if type == :unversioned
+            save_doc bib
+          rescue StandardError => e
+            link = sl.respond_to?(:link) ? sl.link : sl.version_of
+            warn "Error: document #{link} #{e.message}"
+            warn e.backtrace.join("\n")
+          end
+        end
       end
-      query_unversioned_docs.each do |sl|
-        save_doc DataParser.parse(sl, self)
-      rescue StandardError => e
-        warn "Error: document #{sl.version_of} #{e.message}"
-        warn e.backtrace.join("\n")
+      @index.sort!.save
+    end
+
+    #
+    # Add hasEdition relations form previous parsed document
+    #
+    # @param [RelatonW3c::W3cBibliographicItem] bib bibligraphic item
+    #
+    def add_has_edition_relation(bib) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity
+      file = file_name bib.docnumber
+      return unless File.exist? file
+
+      b = case @format
+          when "xml" then XMLParser.from_xml(File.read(file, encoding: "UTF-8"))
+          when "yaml"
+            hash = YAML.load_file(file)
+            W3cBibliographicItem.from_hash(hash)
+          when "bibxml" then BibXMLParser.parse File.read(file, encoding: "UTF-8")
+          end
+      b.relation.each do |r|
+        same_edition = bib.relation.detect { |r2| same_edition?(r, r2) }
+        bib.relation << r unless same_edition
       end
-      Dir[File.expand_path("../../data/*", __dir__)].each do |file|
-        xml = File.read file, encoding: "UTF-8"
-        save_doc BibXMLParser.parse(xml), warn_duplicate: false
-      rescue StandardError => e
-        warn "Error: document #{file} #{e.message}"
-        warn e.backtrace.join("\n")
+    end
+
+    #
+    # Compare two relations
+    #
+    # @param [RelatonW3c::W3cBibliographicItem] rel1 relation 1
+    # @param [RelatonW3c::W3cBibliographicItem] rel2 relation 2
+    #
+    # @return [Boolean] true if relations are same
+    #
+    def same_edition?(rel1, rel2)
+      return false unless rel1.type == "hasEdition" && rel1.type == rel2.type
+
+      ids1 = rel1.bibitem.docidentifier.map(&:id)
+      ids2 = rel2.bibitem.docidentifier.map(&:id)
+      (ids1 & ids2).any?
+    end
+
+    #
+    # Yield fetching for each dataset
+    #
+    # @param [String] source source name "w3c-tr-archive" or "w3c-rdf"
+    #
+    # @yield [RDF::Repository] RDF repository
+    #
+    def each_dataset(source, &_block) # rubocop:disable Metrics/MethodLength
+      case source
+      when "w3c-tr-archive"
+        Dir["w3c-tr-archive/*.rdf"].map do |f|
+          @files = []
+          yield RDF::Repository.load(f)
+        end
+      when "w3c-rdf"
+        @files = []
+        rdf = RDF::Repository.load("http://www.w3.org/2002/01/tr-automation/tr.rdf")
+        yield rdf
+        # parse_static_dataset
       end
-      @index.sort!.save
     end
 
+    #
+    # Parse static dataset
+    #
+    # def parse_static_dataset
+    #   Dir[File.expand_path("../../data/*", __dir__)].each do |file|
+    #     xml = File.read file, encoding: "UTF-8"
+    #     save_doc BibXMLParser.parse(xml), warn_duplicate: false
+    #   rescue StandardError => e
+    #     warn "Error: document #{file} #{e.message}"
+    #     warn e.backtrace.join("\n")
+    #   end
+    # end
+
     #
     # Query RDF source for versioned documents
     #
     # @return [RDF::Query::Solutions] query results
     #
-    def query_versioned_docs # rubocop:disable Metrics/MethodLength
+    def query_versioned_docs(rdf)
       sse = SPARQL.parse(%(
         PREFIX : <http://www.w3.org/2001/02pd/rec54#>
         PREFIX dc: <http://purl.org/dc/elements/1.1/>
         PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
-        # PREFIX mat: <http://www.w3.org/2002/05/matrix/vocab#>
         PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
-        SELECT ?link ?title ?date ?version_of
-        WHERE {
-          ?link dc:title ?title ; dc:date ?date ; doc:versionOf ?version_of .
-        }
+        SELECT ?link ?title ?date
+        WHERE { ?link dc:title ?title ; dc:date ?date . }
       ))
-      data.query sse
+      rdf.query sse
     end
 
     #
@@ -92,13 +157,16 @@ module RelatonW3c
     #
     # @return [Array<RDF::Query::Solution>] query results
     #
-    def query_unversioned_docs
+    def query_unversioned_docs(rdf)
       sse = SPARQL.parse(%(
         PREFIX doc: <http://www.w3.org/2000/10/swap/pim/doc#>
         SELECT ?version_of
-        WHERE { ?x doc:versionOf ?version_of . }
+        WHERE {
+          ?link doc:versionOf ?version_of .
+          FILTER ( isURI(?link) && isURI(?version_of) && ?link != ?version_of )
+        }
       ))
-      data.query(sse).uniq &:version_of
+      rdf.query(sse).uniq { |s| s.version_of.to_s.sub(/^https?:\/\//, "").sub(/\/$/, "") }
     end
 
     #