metanorma 2.4.3 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +13 -1
- data/lib/metanorma/collection/artifact_store.rb +142 -0
- data/lib/metanorma/collection/filelookup/filelookup.rb +7 -21
- data/lib/metanorma/collection/filelookup/filelookup_sectionsplit.rb +9 -14
- data/lib/metanorma/collection/filelookup/utils.rb +7 -48
- data/lib/metanorma/collection/manifest/manifest.rb +6 -0
- data/lib/metanorma/collection/renderer/fileparse.rb +32 -5
- data/lib/metanorma/collection/renderer/fileprocess.rb +74 -13
- data/lib/metanorma/collection/renderer/renderer.rb +67 -19
- data/lib/metanorma/collection/sectionsplit/sectionsplit.rb +41 -10
- data/lib/metanorma/collection/util/util.rb +16 -0
- data/lib/metanorma/version.rb +1 -1
- data/metanorma.gemspec +1 -1
- data/plans/collection-rendering-architecture-review.md +516 -0
- metadata +7 -9
- data/lib/metanorma/collection/filelookup/base.rb +0 -43
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e9167ec4dc502678a508ea35bb8ed81d3914487b0dd2f2562b2e26bd2803faff
|
|
4
|
+
data.tar.gz: cfe251c6ed0436df5b7f9eba6c01e368fb0bee55651366a83a949f403f2b10bc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5fe17ebf90571ce884b547f38a8736947d749ad9ec45a7dfcc9e8bb1aef91f2b0e2e284d509d70da150cb806c402945555d698c6f06e03006a46500e4648707e
|
|
7
|
+
data.tar.gz: 6bf8355bd20b904f7fc7960aadbbbb6363e052b0c7d83689562f0f1fea4811a0785006f24e1766fbf5d354a44424b37f7ea749328229b8e0af925ab90288176f
|
data/.rubocop.yml
CHANGED
|
@@ -3,8 +3,20 @@
|
|
|
3
3
|
inherit_from:
|
|
4
4
|
- https://raw.githubusercontent.com/riboseinc/oss-guides/main/ci/rubocop.yml
|
|
5
5
|
|
|
6
|
+
# Rubocop plugins enabled centrally so every metanorma-org gem picks them up
|
|
7
|
+
# on cimas sync — best practice belongs at the shared-template layer, not
|
|
8
|
+
# per-repo. Per ronaldtse feedback on metanorma/ci#332.
|
|
9
|
+
plugins:
|
|
10
|
+
- rubocop-rspec
|
|
11
|
+
- rubocop-performance
|
|
12
|
+
- rubocop-rake
|
|
13
|
+
|
|
6
14
|
# local repo-specific modifications
|
|
7
15
|
# ...
|
|
8
16
|
|
|
9
17
|
AllCops:
|
|
10
|
-
|
|
18
|
+
# 3.3 matches the org-wide minimum being pushed via #274 (Raise minimum
|
|
19
|
+
# Ruby version to 3.3 due to EOL of 3.2 on 2026-03-31). Was 3.4 before
|
|
20
|
+
# this commit — 3.4 was above the org's stated minimum and would have
|
|
21
|
+
# applied Rubocop rules that fail-close on gems still targeting 3.3.
|
|
22
|
+
TargetRubyVersion: 3.3
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "digest"
|
|
5
|
+
|
|
6
|
+
module Metanorma
|
|
7
|
+
class Collection
|
|
8
|
+
# Durable, content-addressed store for staged collection-build artefacts.
|
|
9
|
+
#
|
|
10
|
+
# Enables incremental (batched) builds and resumption across runs: an
|
|
11
|
+
# artefact is named +<docid-slug>.<content-hash>.<stage>.<format>+, so a
|
|
12
|
+
# document already built with identical inputs is detected by filename alone
|
|
13
|
+
# and reused, and an interrupted run resumes by recomputing which artefacts
|
|
14
|
+
# are already present.
|
|
15
|
+
#
|
|
16
|
+
# Reuse is keyed on the caller-supplied content hash of the document's
|
|
17
|
+
# *input closure* (source + resolved includes + templates + referenced
|
|
18
|
+
# schemas + tool versions + options) -- never on file presence alone. See
|
|
19
|
+
# metanorma/iso-10303#455 for why presence-keyed reuse of compiled XML is
|
|
20
|
+
# unsound; the store deliberately requires a content hash it does not infer.
|
|
21
|
+
#
|
|
22
|
+
# Disabled by default: the collection renderer uses NullArtifactStore unless
|
|
23
|
+
# an incremental build is explicitly requested (opt-in, never default).
|
|
24
|
+
class ArtifactStore
|
|
25
|
+
DEFAULT_DIRNAME = ".metanorma-collection-cache"
|
|
26
|
+
|
|
27
|
+
# The artefact-stage vocabulary, and the one non-redundant property each
|
|
28
|
+
# stage carries: the format it is serialised as (also its file extension).
|
|
29
|
+
# THIS IS ITS HOME: every producer and consumer names its stage from this
|
|
30
|
+
# table by symbol, never a bare string literal elsewhere; an unknown stage
|
|
31
|
+
# raises (KeyError via +fetch+), so the vocabulary cannot drift outside
|
|
32
|
+
# this file. The stage's *name* is the filename token, so it is not stored
|
|
33
|
+
# here -- that would just restate the key. The stages trace the pipeline
|
|
34
|
+
# (isolated compile -> index -> reinflate); each role is documented inline:
|
|
35
|
+
STAGE_FORMATS = {
|
|
36
|
+
# Compiled Semantic XML, unresolved cross-document repo:() references
|
|
37
|
+
# PRESERVED as deterministic stubs (neither stripped nor resolved). The
|
|
38
|
+
# unit of an isolated per-document compile; a pure function of the
|
|
39
|
+
# document's input closure, hence content-addressable and reusable.
|
|
40
|
+
semantic: "xml",
|
|
41
|
+
# This document's contribution to the global anchor index: its anchors
|
|
42
|
+
# and ids (type => {label/UUID => anchor-id}). Aggregated across all
|
|
43
|
+
# documents to build the anchor -> owning-document index used at
|
|
44
|
+
# reinflation.
|
|
45
|
+
anchors: "json",
|
|
46
|
+
# Reinflated Presentation XML: the preserved stubs resolved against the
|
|
47
|
+
# global index into real cross-document links. Shared source for the PDF
|
|
48
|
+
# and Word collection outputs, and per-document for HTML.
|
|
49
|
+
presentation: "xml",
|
|
50
|
+
}.freeze
|
|
51
|
+
|
|
52
|
+
attr_reader :dir
|
|
53
|
+
|
|
54
|
+
def initialize(dir)
|
|
55
|
+
@dir = dir
|
|
56
|
+
FileUtils.mkdir_p(@dir)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# <docid-slug>.<content-hash>.<stage>.<format>
|
|
60
|
+
# +stage+ must be a key of STAGE_FORMATS; an unknown stage raises KeyError.
|
|
61
|
+
def path(docid, content_hash, stage)
|
|
62
|
+
format = STAGE_FORMATS.fetch(stage)
|
|
63
|
+
File.join(@dir, "#{slug(docid)}.#{content_hash}.#{stage}.#{format}")
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Is this exact (document, input-closure, stage) artefact already present?
|
|
67
|
+
# This is the resumption / skip predicate: a hit is byte-identical to a
|
|
68
|
+
# fresh build because the hash covers every input that determines output.
|
|
69
|
+
def key?(docid, content_hash, stage)
|
|
70
|
+
File.exist?(path(docid, content_hash, stage))
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def read(docid, content_hash, stage)
|
|
74
|
+
f = path(docid, content_hash, stage)
|
|
75
|
+
File.exist?(f) ? File.read(f, encoding: "UTF-8") : nil
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Idempotent write: the same (docid, content_hash, stage) always maps to
|
|
79
|
+
# the same path and the content is a pure function of the hash, so a re-run
|
|
80
|
+
# overwrites with identical bytes -- no side effects on repeat.
|
|
81
|
+
def write(docid, content_hash, stage, content)
|
|
82
|
+
f = path(docid, content_hash, stage)
|
|
83
|
+
File.write(f, content, encoding: "UTF-8")
|
|
84
|
+
f
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Remove every stored version of this document whose content hash is not
|
|
88
|
+
# +keep_hash+ -- the superseded artefacts left behind when the document's
|
|
89
|
+
# content changed. Keeps exactly one (current) version per document per
|
|
90
|
+
# stage, bounding disk to current state without losing the cache the next
|
|
91
|
+
# build resumes from.
|
|
92
|
+
def prune_superseded(docid, keep_hash)
|
|
93
|
+
STAGE_FORMATS.each do |stage, ext|
|
|
94
|
+
keep = path(docid, keep_hash, stage)
|
|
95
|
+
Dir[File.join(@dir, "#{slug(docid)}.*.#{stage}.#{ext}")].each do |f|
|
|
96
|
+
f == keep or File.delete(f)
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Wipe the entire store. The escape hatch when inputs the content hash
|
|
102
|
+
# cannot see -- imported EXPRESS schemas, templates, included files -- have
|
|
103
|
+
# changed, so the cache can no longer be trusted; the next build recompiles
|
|
104
|
+
# everything. Tracking such shared-input changes is the collection
|
|
105
|
+
# maintainer's responsibility, not the build's.
|
|
106
|
+
def clear
|
|
107
|
+
FileUtils.rm_rf(@dir)
|
|
108
|
+
FileUtils.mkdir_p(@dir)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# SHA256 over the input closure, truncated to 16 hex (64 bits -- ample for
|
|
112
|
+
# a per-collection store, and short enough to stay well inside filename
|
|
113
|
+
# length limits). Each input is hashed to a fixed-width digest and the
|
|
114
|
+
# digests concatenated before the final hash, so no two distinct closures
|
|
115
|
+
# collide on the pre-truncation digest. The caller assembles the closure;
|
|
116
|
+
# the store does not infer it.
|
|
117
|
+
def self.content_hash(*inputs)
|
|
118
|
+
digests = inputs.map { |i| Digest::SHA256.hexdigest(i.to_s) }
|
|
119
|
+
Digest::SHA256.hexdigest(digests.join)[0, 16]
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
private
|
|
123
|
+
|
|
124
|
+
def slug(docid)
|
|
125
|
+
docid.to_s.gsub(/[^A-Za-z0-9._-]+/, "-")
|
|
126
|
+
.gsub(/-{2,}/, "-").gsub(/\A-|-\z/, "")
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Null object: the store disabled (the default). Never reports a hit, never
|
|
131
|
+
# writes. Lets the renderer treat "no incremental build" as the no-store
|
|
132
|
+
# path without a conditional at every call site (open/closed).
|
|
133
|
+
class NullArtifactStore
|
|
134
|
+
def path(*_args) = nil
|
|
135
|
+
def key?(*_args) = false
|
|
136
|
+
def read(*_args) = nil
|
|
137
|
+
def write(*_args) = nil
|
|
138
|
+
def prune_superseded(*_args) = nil
|
|
139
|
+
def clear(*_args) = nil
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
@@ -3,7 +3,6 @@ require "htmlentities"
|
|
|
3
3
|
require "metanorma-utils"
|
|
4
4
|
require "marcel"
|
|
5
5
|
require_relative "filelookup_sectionsplit"
|
|
6
|
-
require_relative "base"
|
|
7
6
|
require_relative "utils"
|
|
8
7
|
|
|
9
8
|
module Metanorma
|
|
@@ -57,17 +56,18 @@ module Metanorma
|
|
|
57
56
|
entry = file_entry(manifest, k, idx) or return
|
|
58
57
|
bibdata_process(entry, i)
|
|
59
58
|
bibitem_process(entry)
|
|
60
|
-
@files[
|
|
59
|
+
@files[entry_key(i)] = entry
|
|
61
60
|
end
|
|
62
61
|
|
|
63
62
|
def read_file_idents(manifest)
|
|
64
63
|
id = manifest.identifier
|
|
65
|
-
sanitised_id =
|
|
64
|
+
sanitised_id =
|
|
65
|
+
entry_key(@isodoc.docid_prefix("", manifest.identifier.dup))
|
|
66
66
|
# if manifest.bibdata and # NO, DO NOT FISH FOR THE GENUINE IDENTIFIER IN BIBDATA
|
|
67
67
|
# d = manifest.bibdata.docidentifier.detect { |x| x.primary } ||
|
|
68
68
|
# manifest.bibdata.docidentifier.first
|
|
69
69
|
# k = d.id
|
|
70
|
-
# i =
|
|
70
|
+
# i = entry_key(@isodoc.docid_prefix(d.type, d.id.dup))
|
|
71
71
|
# end
|
|
72
72
|
[id, sanitised_id]
|
|
73
73
|
end
|
|
@@ -158,24 +158,10 @@ module Metanorma
|
|
|
158
158
|
end
|
|
159
159
|
end
|
|
160
160
|
|
|
161
|
-
# Substitute special strings in filename patterns
|
|
162
|
-
#
|
|
163
|
-
# @param options [Hash] substitution values
|
|
164
|
-
# @option options [Integer] :document_num document index
|
|
165
|
-
# @option options [String] :basename filename without extension
|
|
166
|
-
# @option options [String] :basename_legacy full filename with extension
|
|
167
|
-
# @option options [Integer] :sectionsplit_num sectionsplit index
|
|
161
|
+
# Substitute special strings in filename patterns.
|
|
162
|
+
# See Util.substitute_filename_pattern.
|
|
168
163
|
def substitute_filename_pattern(pattern, options = {})
|
|
169
|
-
pattern
|
|
170
|
-
result = pattern.dup
|
|
171
|
-
options[:document_num] and
|
|
172
|
-
result.gsub!(/\{document-num\}/, options[:document_num].to_s)
|
|
173
|
-
result.gsub!(/\{basename\}/, options[:basename]) if options[:basename]
|
|
174
|
-
options[:basename_legacy] and
|
|
175
|
-
result.gsub!(/\{basename_legacy\}/, options[:basename_legacy])
|
|
176
|
-
options[:sectionsplit_num] and
|
|
177
|
-
result.gsub!(/\{sectionsplit-num\}/, options[:sectionsplit_num].to_s)
|
|
178
|
-
result
|
|
164
|
+
Util.substitute_filename_pattern(pattern, options)
|
|
179
165
|
end
|
|
180
166
|
|
|
181
167
|
# TODO make the output file location reflect source location universally,
|
|
@@ -18,7 +18,6 @@ module Metanorma
|
|
|
18
18
|
# Save the original out_path before it gets modified
|
|
19
19
|
original_out_path = @files[key][:out_path]
|
|
20
20
|
s, sectionsplit_manifest = sectionsplit(key)
|
|
21
|
-
# section_split_instance_threads(s, manifest, key)
|
|
22
21
|
s.each_with_index do |f1, i|
|
|
23
22
|
add_section_split_instance(f1, manifest, key, i)
|
|
24
23
|
end
|
|
@@ -29,18 +28,6 @@ module Metanorma
|
|
|
29
28
|
original_out_path
|
|
30
29
|
end
|
|
31
30
|
|
|
32
|
-
def section_split_instance_threads(s, manifest, key)
|
|
33
|
-
@mutex = Mutex.new
|
|
34
|
-
pool = Concurrent::FixedThreadPool.new(4)
|
|
35
|
-
s.each_with_index do |f1, i|
|
|
36
|
-
pool.post do
|
|
37
|
-
add_section_split_instance(f1, manifest, key, i)
|
|
38
|
-
end
|
|
39
|
-
end
|
|
40
|
-
pool.shutdown
|
|
41
|
-
pool.wait_for_termination
|
|
42
|
-
end
|
|
43
|
-
|
|
44
31
|
def cleanup_section_split_instance(key, manifest, original_out_path)
|
|
45
32
|
# Delete the sectionsplit index.html from source directory after it's copied to output
|
|
46
33
|
@files_to_delete << manifest["#{key}:index.html"][:ref]
|
|
@@ -54,6 +41,14 @@ module Metanorma
|
|
|
54
41
|
end
|
|
55
42
|
# @files[key].delete(:ids).delete(:anchors)
|
|
56
43
|
@files[key][:indirect_key] = @sectionsplit.key
|
|
44
|
+
# sectionsplit is HTML-only; hand the parent its whole Presentation XML
|
|
45
|
+
# so the collection PDF/presentation concatenation inlines the document
|
|
46
|
+
# intact (raw_file) instead of degrading to a bibdata-only, namespace-
|
|
47
|
+
# less stub. file_compile early-returns for sectionsplit, so this
|
|
48
|
+
# :outputs entry is what concatenate1 consumes.
|
|
49
|
+
# https://github.com/metanorma/iso-10303/issues/208
|
|
50
|
+
@files[key][:outputs] =
|
|
51
|
+
{ presentation: @sectionsplit.whole_presentation_file }
|
|
57
52
|
end
|
|
58
53
|
|
|
59
54
|
def add_section_split_cover(manifest, sectionsplit_manifest, ident)
|
|
@@ -134,7 +129,7 @@ module Metanorma
|
|
|
134
129
|
# file[:url] contains full path with directory for HTML output, but XML is basename only
|
|
135
130
|
xml_basename = File.basename(file[:url])
|
|
136
131
|
presfile = File.join(File.dirname(@files[key][:ref]), xml_basename)
|
|
137
|
-
newkey =
|
|
132
|
+
newkey = entry_key("#{key.strip} #{file[:title]}")
|
|
138
133
|
xml = Nokogiri::XML(File.read(presfile), &:huge)
|
|
139
134
|
[presfile, newkey, xml]
|
|
140
135
|
end
|
|
@@ -1,56 +1,12 @@
|
|
|
1
1
|
module Metanorma
|
|
2
2
|
class Collection
|
|
3
3
|
class FileLookup
|
|
4
|
-
# Also parse all ids in doc (including ones which won't be xref targets)
|
|
5
|
-
def read_ids(xml)
|
|
6
|
-
ret = {}
|
|
7
|
-
xml.traverse do |x|
|
|
8
|
-
x.text? and next
|
|
9
|
-
x["id"] and ret[x["id"]] = true
|
|
10
|
-
end
|
|
11
|
-
ret
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
# map locality type and label (e.g. "clause" "1") to id = anchor for
|
|
15
|
-
# a document
|
|
16
|
-
# Note: will only key clauses, which have unambiguous reference label in
|
|
17
|
-
# locality. Notes, examples etc with containers are just plunked against
|
|
18
|
-
# UUIDs, so that their IDs can at least be registered to be tracked
|
|
19
|
-
# as existing.
|
|
20
|
-
def read_anchors(xml)
|
|
21
|
-
xrefs = @isodoc.xref_init(@lang, @script, @isodoc, @isodoc.i18n,
|
|
22
|
-
{ locale: @locale })
|
|
23
|
-
xrefs.parse xml
|
|
24
|
-
xrefs.get.each_with_object({}) do |(k, v), ret|
|
|
25
|
-
read_anchors1(k, v, ret)
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def read_anchors1(key, val, ret)
|
|
30
|
-
val[:type] ||= "clause"
|
|
31
|
-
ret[val[:type]] ||= {}
|
|
32
|
-
index = if val[:container] || val[:label].nil? || val[:label].empty?
|
|
33
|
-
UUIDTools::UUID.random_create.to_s
|
|
34
|
-
else val[:label].gsub(%r{<[^<>]+>}, "")
|
|
35
|
-
end
|
|
36
|
-
ret[val[:type]][index] = key
|
|
37
|
-
v = val[:value] and ret[val[:type]][v.gsub(%r{<[^<>]+>}, "")] = key
|
|
38
|
-
end
|
|
39
|
-
|
|
40
4
|
def anchors_lookup(anchors)
|
|
41
5
|
anchors.values.each_with_object({}) do |v, m|
|
|
42
6
|
v.each_value { |v1| m[v1] = true }
|
|
43
7
|
end
|
|
44
8
|
end
|
|
45
9
|
|
|
46
|
-
# return citation url for file
|
|
47
|
-
# @param doc [Boolean] I am a Metanorma document,
|
|
48
|
-
# so my URL should end with html or pdf or whatever
|
|
49
|
-
def url(ident, options)
|
|
50
|
-
data = get(ident)
|
|
51
|
-
data[:url] || targetfile(data, options)[1]
|
|
52
|
-
end
|
|
53
|
-
|
|
54
10
|
# are references to the file to be linked to a file in the collection,
|
|
55
11
|
# or externally? Determines whether file suffix anchors are to be used
|
|
56
12
|
def url?(ident)
|
|
@@ -58,7 +14,10 @@ module Metanorma
|
|
|
58
14
|
data[:url]
|
|
59
15
|
end
|
|
60
16
|
|
|
61
|
-
|
|
17
|
+
# Normalise an identifier to its @files hash key: decode entities, squeeze
|
|
18
|
+
# whitespace, and strip a leading "metanorma-collection " prefix. Distinct
|
|
19
|
+
# from Util::key, which does NOT strip that prefix.
|
|
20
|
+
def entry_key(ident)
|
|
62
21
|
@c.decode(ident).gsub(/(\p{Zs})+/, " ")
|
|
63
22
|
.sub(/^metanorma-collection /, "")
|
|
64
23
|
end
|
|
@@ -68,13 +27,13 @@ module Metanorma
|
|
|
68
27
|
end
|
|
69
28
|
|
|
70
29
|
def get(ident, attr = nil)
|
|
71
|
-
if attr then @files[
|
|
72
|
-
else @files[
|
|
30
|
+
if attr then @files[entry_key(ident)][attr]
|
|
31
|
+
else @files[entry_key(ident)]
|
|
73
32
|
end
|
|
74
33
|
end
|
|
75
34
|
|
|
76
35
|
def set(ident, attr, value)
|
|
77
|
-
@files[
|
|
36
|
+
@files[entry_key(ident)][attr] = value
|
|
78
37
|
end
|
|
79
38
|
|
|
80
39
|
def each
|
|
@@ -20,6 +20,12 @@ module Metanorma
|
|
|
20
20
|
@config = manifest_postprocess(config)
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
# Ordered pipeline of in-place tree-walks over the manifest; the order is
|
|
24
|
+
# load-bearing. expand_yaml must run early because it grafts nested
|
|
25
|
+
# `fileref` manifests into this tree, so every later pass sees the full
|
|
26
|
+
# entry set; bibdata sets @lang/@script used downstream; compile_adoc
|
|
27
|
+
# turns .adoc sources into .xml before filexist verifies them. Each pass
|
|
28
|
+
# recurses the manifest independently (see the architecture review plan).
|
|
23
29
|
def manifest_postprocess(config)
|
|
24
30
|
manifest_bibdata(config)
|
|
25
31
|
manifest_expand_yaml(config, @dir)
|
|
@@ -14,24 +14,46 @@ module Metanorma
|
|
|
14
14
|
# @param internal_refs [Hash{String=>Hash{String=>String}] schema name to
|
|
15
15
|
# anchor to filename
|
|
16
16
|
# @return [String] XML content
|
|
17
|
+
# Each pass below mutates the SAME Nokogiri tree in place; the inline
|
|
18
|
+
# comments state the post-condition each pass guarantees on exit. The
|
|
19
|
+
# contract between passes is "the tree is now in state X" -- there is no
|
|
20
|
+
# intermediate typed representation, so the ordering is load-bearing.
|
|
21
|
+
# `sso` truthy (sectionsplit_output) => `xml` is Presentation XML and the
|
|
22
|
+
# semantic-only passes are skipped; otherwise `xml` is Semantic XML.
|
|
23
|
+
# See the pipeline map in the collection architecture review plan.
|
|
17
24
|
def update_xrefs(file, docid, internal_refs)
|
|
18
25
|
xml, sso = update_xrefs_prep(file, docid)
|
|
19
|
-
|
|
26
|
+
# post: repository docidentifiers supplied on bibitems naming a
|
|
27
|
+
# collection file; `sso` resolved (Presentation vs Semantic).
|
|
28
|
+
@nested || sso || @reinflate or
|
|
20
29
|
Metanorma::Collection::XrefProcess::xref_process(xml, xml, nil, docid,
|
|
21
30
|
@isodoc, sso)
|
|
31
|
+
# post (semantic only): intra-doc xref/eref turned into internal erefs;
|
|
32
|
+
# repo bibitems copied; indirect bibliography inserted.
|
|
22
33
|
@ncnames = {}
|
|
23
34
|
@nested or update_indirect_refs_to_docs(xml, docid, internal_refs, sso)
|
|
24
|
-
@
|
|
35
|
+
# post: bibitem[@type='internal'] anchors resolved to the containing
|
|
36
|
+
# collection document.
|
|
37
|
+
@reinflate or @files.add_document_suffix(docid, xml)
|
|
38
|
+
# post: every @id/@anchor namespaced with this doc's NCName suffix, so
|
|
39
|
+
# concatenated documents do not collide.
|
|
25
40
|
@nested or update_sectionsplit_refs_to_docs(xml, docid, internal_refs,
|
|
26
41
|
sso)
|
|
42
|
+
# post: erefs targeting a sectionsplit document point at the specific
|
|
43
|
+
# split section file that holds the anchor.
|
|
27
44
|
update_direct_refs_to_docs(xml, docid, sso)
|
|
45
|
+
# post: repo(current-metanorma-collection/x) bibitems replaced in situ
|
|
46
|
+
# with resolved bibdata + citation URI; their erefs become hyperlinks.
|
|
28
47
|
::Metanorma::Collection::Util::hide_refs(xml)
|
|
48
|
+
# post: references containers with only hidden bibitems flagged hidden.
|
|
29
49
|
sso and eref2link(xml, sso)
|
|
30
|
-
@nested or svgmap_resolve(xml, docid, sso)
|
|
50
|
+
@nested || @reinflate or svgmap_resolve(xml, docid, sso)
|
|
31
51
|
xml.to_xml
|
|
32
52
|
end
|
|
33
53
|
|
|
34
|
-
## sso
|
|
54
|
+
## sso (sectionsplit_output) truthy => Presentation XML, else Semantic.
|
|
55
|
+
## This is the single discriminator that decides which grammar `xml`
|
|
56
|
+
## carries and therefore which passes in update_xrefs run.
|
|
35
57
|
def update_xrefs_prep(file, docid)
|
|
36
58
|
docxml = file.is_a?(String) ? Nokogiri::XML(file, &:huge) : file
|
|
37
59
|
supply_repo_ids(docxml)
|
|
@@ -121,8 +143,13 @@ presxml)
|
|
|
121
143
|
# Do not do this if this is a sectionsplit collection or a nested manifest.
|
|
122
144
|
# Return false if bibitem is not to be further processed
|
|
123
145
|
def strip_unresolved_repo_erefs(_document_id, bib_docid, erefs, bibitem)
|
|
124
|
-
%r{^current-metanorma-collection/(?!Missing:)}.match?(bib_docid.text)
|
|
146
|
+
if %r{^current-metanorma-collection/(?!Missing:)}.match?(bib_docid.text)
|
|
147
|
+
# Isolated build: keep the cross-document ref as a stub (its bibitemid
|
|
148
|
+
# and repository docidentifier intact) for a later reinflation pass to
|
|
149
|
+
# relink, rather than resolving it now against an absent target.
|
|
150
|
+
@preserve_unresolved and return false
|
|
125
151
|
return true
|
|
152
|
+
end
|
|
126
153
|
@nested and return false
|
|
127
154
|
erefs[bibitem["id"]]&.each { |x| x.parent and strip_eref(x) }
|
|
128
155
|
false
|
|
@@ -4,6 +4,8 @@ require "isodoc"
|
|
|
4
4
|
require "metanorma-utils"
|
|
5
5
|
require_relative "fileparse"
|
|
6
6
|
require_relative "filelocation"
|
|
7
|
+
require_relative "../artifact_store"
|
|
8
|
+
require "json"
|
|
7
9
|
|
|
8
10
|
module Metanorma
|
|
9
11
|
class Collection
|
|
@@ -67,8 +69,11 @@ module Metanorma
|
|
|
67
69
|
if @files.get(ident, :attachment) then copy_file_to_dest(ident)
|
|
68
70
|
else
|
|
69
71
|
file, fname = @files.targetfile_id(ident, read: true)
|
|
72
|
+
@preserve_unresolved && member_stored?(ident, file) and next
|
|
70
73
|
warn "\n\n\n\n\nProcess #{fname}: #{DateTime.now.strftime('%H:%M:%S')}"
|
|
71
74
|
collection_xml = update_xrefs(file, ident, internal_refs)
|
|
75
|
+
@preserve_unresolved and
|
|
76
|
+
store_member_artefacts(ident, file, collection_xml)
|
|
72
77
|
# Strip .xml or .html extension, but NOT section numbers like .0
|
|
73
78
|
fname_base = File.basename(fname)
|
|
74
79
|
collection_filename = fname_base
|
|
@@ -83,6 +88,31 @@ module Metanorma
|
|
|
83
88
|
end
|
|
84
89
|
end
|
|
85
90
|
|
|
91
|
+
# Persist an isolated member's stub-bearing Semantic XML and its anchor
|
|
92
|
+
# contribution to the content-addressed store, so a later reinflation pass
|
|
93
|
+
# can relink the stubs and a re-run can resume without recompiling.
|
|
94
|
+
# Within-campaign content key for a member. (Cross-build soundness needs
|
|
95
|
+
# the full input closure -- source + includes + templates + schemas +
|
|
96
|
+
# versions; see the plan gotcha. Here the source bytes + tool version
|
|
97
|
+
# suffice because inputs are frozen across a single build campaign.)
|
|
98
|
+
def member_content_hash(source)
|
|
99
|
+
ArtifactStore.content_hash(source, ::Metanorma::VERSION)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Resume predicate: is this member already compiled and stored for its
|
|
103
|
+
# current input content? If so the isolated compile can skip it.
|
|
104
|
+
def member_stored?(ident, source)
|
|
105
|
+
@artifact_store.key?(ident, member_content_hash(source), :semantic)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def store_member_artefacts(ident, source, semantic_xml)
|
|
109
|
+
hash = member_content_hash(source)
|
|
110
|
+
@artifact_store.write(ident, hash, :semantic, semantic_xml)
|
|
111
|
+
@artifact_store.write(ident, hash, :anchors,
|
|
112
|
+
JSON.generate(@files.get(ident, :anchors) || {}))
|
|
113
|
+
@keep_cache or @artifact_store.prune_superseded(ident, hash)
|
|
114
|
+
end
|
|
115
|
+
|
|
86
116
|
# gather internal bibitem references
|
|
87
117
|
def gather_internal_refs
|
|
88
118
|
@files.keys.each_with_object({}) do |i, refs|
|
|
@@ -183,22 +213,53 @@ module Metanorma
|
|
|
183
213
|
def update_bibitem_prep(bib, identifier)
|
|
184
214
|
docid = get_bibitem_docid(bib, identifier) or return [nil, nil]
|
|
185
215
|
newbib = dup_bibitem(docid, bib)
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
# otherwise convert :out_path to HTML (before compilation)
|
|
190
|
-
current_html = @files.get(identifier, :outputs)&.dig(:html)
|
|
191
|
-
if !current_html && (out_path = @files.get(identifier, :out_path))
|
|
192
|
-
# Convert .xml to .html, following same logic as ref_file_xml2html
|
|
193
|
-
current_html = if out_path.end_with?(".xml")
|
|
194
|
-
out_path.sub(/\.xml$/, ".html")
|
|
195
|
-
else
|
|
196
|
-
"#{out_path}.html"
|
|
197
|
-
end
|
|
198
|
-
end
|
|
216
|
+
attachment = @files.get(docid, :attachment)
|
|
217
|
+
url = @files.url(docid, relative: true, doc: !attachment)
|
|
218
|
+
current_html = referencing_html_location(identifier, attachment)
|
|
199
219
|
url = make_relative_path(current_html, url) if current_html
|
|
200
220
|
[newbib, url]
|
|
201
221
|
end
|
|
222
|
+
|
|
223
|
+
# Location of the HTML that will carry the cross-reference to the target,
|
|
224
|
+
# used as the base for relativising the attachment/citation URL.
|
|
225
|
+
#
|
|
226
|
+
# Normally this is the referencing document's own output (:outputs[:html]
|
|
227
|
+
# after compilation, else :out_path converted to .html before it).
|
|
228
|
+
#
|
|
229
|
+
# But when the referencing document is sectionsplit, its content is not
|
|
230
|
+
# emitted at its own out_path: it is split into section files written at
|
|
231
|
+
# the sectionsplit output location (the collection root, unless a
|
|
232
|
+
# sectionsplit_filename directory is configured). Relativising an
|
|
233
|
+
# attachment URL against the unsplit out_path then overshoots -- the
|
|
234
|
+
# ../../ climbs above the split file and the attachment link breaks
|
|
235
|
+
# (https://github.com/metanorma/iso-10303/issues/208). So for attachment
|
|
236
|
+
# targets of a sectionsplit document, base the relative path on the
|
|
237
|
+
# sectionsplit output directory instead.
|
|
238
|
+
def referencing_html_location(identifier, attachment)
|
|
239
|
+
attachment && @files.get(identifier, :sectionsplit) and
|
|
240
|
+
return sectionsplit_ref_html(identifier)
|
|
241
|
+
document_ref_html(identifier)
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# The split section files sit at the sectionsplit output directory: the
|
|
245
|
+
# collection root, or the directory of a configured sectionsplit_filename.
|
|
246
|
+
def sectionsplit_ref_html(identifier)
|
|
247
|
+
d = @files.preserve_directory_structure?(identifier)
|
|
248
|
+
d ? File.join(File.dirname(d), "x.html") : "x.html"
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# Use :outputs[:html] if available (after compilation), otherwise convert
|
|
252
|
+
# :out_path to HTML (before compilation), following ref_file_xml2html.
|
|
253
|
+
def document_ref_html(identifier)
|
|
254
|
+
@files.get(identifier, :outputs)&.dig(:html) ||
|
|
255
|
+
out_path_to_html(@files.get(identifier, :out_path))
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def out_path_to_html(out_path)
|
|
259
|
+
out_path or return nil
|
|
260
|
+
out_path.end_with?(".xml") and return out_path.sub(/\.xml$/, ".html")
|
|
261
|
+
"#{out_path}.html"
|
|
262
|
+
end
|
|
202
263
|
end
|
|
203
264
|
end
|
|
204
265
|
end
|