metanorma 2.5.23 → 2.5.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.devel +1 -1
- data/lib/metanorma/collection/collection.rb +1 -1
- data/lib/metanorma/collection/config/bibdata_parser.rb +136 -0
- data/lib/metanorma/collection/config/converters.rb +3 -2
- data/lib/metanorma/collection/document/document.rb +4 -3
- data/lib/metanorma/collection/renderer/filelocation.rb +27 -21
- data/lib/metanorma/collection/renderer/fileparse.rb +52 -41
- data/lib/metanorma/collection/renderer/fileprocess.rb +180 -41
- data/lib/metanorma/collection/renderer/navigation.rb +28 -9
- data/lib/metanorma/collection/renderer/renderer.rb +15 -2
- data/lib/metanorma/collection/renderer/svg.rb +25 -5
- data/lib/metanorma/collection/sectionsplit/collection.rb +4 -1
- data/lib/metanorma/collection/sectionsplit/sectionsplit.rb +8 -3
- data/lib/metanorma/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cb5842a8eea875101cbaf77d25d62d9fa5ca74f15ed8dcc69273108a95335998
|
|
4
|
+
data.tar.gz: 5042b060a3d433bf0852e1a2dbad3a3b44201438ce875dc7bdb30af98c79eaa3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ff6f596ef437df009705e085713692e51fee114676e4227d912bd7556155b1b8efbdbebb08e51d3f15b1abc1ca6f635f72abc6c245138f4077c1390523583627
|
|
7
|
+
data.tar.gz: 5f2d5554445a316746c08401e5ebc071e4f22132f7f252b225068bc603aff0cf014b69738b11f5341520683fab7d445b2cef7dc7af3f348d4d0d0042160ddb87
|
data/Gemfile.devel
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
require "relaton"
|
|
2
2
|
require "relaton/bib"
|
|
3
|
-
require "relaton/cli"
|
|
4
3
|
require "metanorma-utils"
|
|
5
4
|
require_relative "util/util"
|
|
6
5
|
require_relative "util/disambig_files"
|
|
6
|
+
require_relative "config/bibdata_parser"
|
|
7
7
|
require_relative "config/config"
|
|
8
8
|
require_relative "config/manifest"
|
|
9
9
|
require_relative "helpers"
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "moxml"
|
|
4
|
+
require "relaton"
|
|
5
|
+
require "relaton/bib"
|
|
6
|
+
|
|
7
|
+
module Metanorma
|
|
8
|
+
class Collection
|
|
9
|
+
module Config
|
|
10
|
+
# Collection bibdata ingestion through relaton's library surface.
|
|
11
|
+
#
|
|
12
|
+
# Mirrors the flavor dispatch of the relaton Cli entry points
|
|
13
|
+
# (cli.rb parse_xml, YAMLConvertor convert_single_file): the
|
|
14
|
+
# processor is named by the first docidentifier's type, else by the
|
|
15
|
+
# leading word of its text. The long-term home for this logic is a
|
|
16
|
+
# relaton library-level API (follow-up issue, not filed from here);
|
|
17
|
+
# metanorma must not call the user-facing Cli layer.
|
|
18
|
+
module BibdataParser
|
|
19
|
+
class << self
|
|
20
|
+
# @param xml [String] serialized bibdata or bibitem XML
|
|
21
|
+
# @return [Relaton::Bib::ItemData, nil] nil when the source is not
|
|
22
|
+
# parseable XML
|
|
23
|
+
def from_xml(xml)
|
|
24
|
+
return unless xml
|
|
25
|
+
|
|
26
|
+
doc = Moxml.new.parse(xml)
|
|
27
|
+
plain = without_namespaces(doc)
|
|
28
|
+
if (processor = processor_for_xml(doc))
|
|
29
|
+
processor.from_xml(plain)
|
|
30
|
+
else
|
|
31
|
+
# No-flavor fallback: unparseable sources degrade to a nil
|
|
32
|
+
# bibdata, as in the mirrored Cli method.
|
|
33
|
+
Relaton::Bib::Item.from_xml(plain) rescue nil
|
|
34
|
+
end
|
|
35
|
+
rescue Moxml::ParseError
|
|
36
|
+
nil
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @param content [Hash] bibdata in YAML form
|
|
40
|
+
# @return [Relaton::Bib::ItemData]
|
|
41
|
+
def from_yaml(content)
|
|
42
|
+
yaml = content.to_yaml
|
|
43
|
+
if (processor = processor_for_yaml(content))
|
|
44
|
+
begin
|
|
45
|
+
processor.from_yaml(yaml)
|
|
46
|
+
rescue RuntimeError
|
|
47
|
+
Relaton::Bib::Item.from_yaml(yaml)
|
|
48
|
+
end
|
|
49
|
+
else
|
|
50
|
+
Relaton::Bib::Item.from_yaml(yaml)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
# The processor named by the first docidentifier's type, else by
|
|
57
|
+
# the first word of its text.
|
|
58
|
+
def processor_for_xml(doc)
|
|
59
|
+
docid = doc.xpath("//*[local-name()='docidentifier']").first
|
|
60
|
+
proc = get_proc(docid)
|
|
61
|
+
return proc if proc
|
|
62
|
+
|
|
63
|
+
Relaton::Db::Registry.instance
|
|
64
|
+
.by_type(docid&.text&.match(/^\w+/)&.to_s)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def get_proc(docid)
|
|
68
|
+
type = docid&.[]("type")
|
|
69
|
+
return unless type
|
|
70
|
+
|
|
71
|
+
Relaton::Db::Registry.instance.by_type(type)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The processor named by `ext.flavor`, else by the first
|
|
75
|
+
# docidentifier's type / leading word of its content.
|
|
76
|
+
def processor_for_yaml(content)
|
|
77
|
+
flavor = content.dig("ext", "flavor") ||
|
|
78
|
+
yaml_doctype(content["docidentifier"])
|
|
79
|
+
return unless flavor
|
|
80
|
+
|
|
81
|
+
Relaton::Db::Registry.instance.by_type(flavor)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def yaml_doctype(docid)
|
|
85
|
+
did = docid.is_a?(Array) ? docid.fetch(0) : docid
|
|
86
|
+
return unless did
|
|
87
|
+
|
|
88
|
+
did["type"] || did.fetch("content")&.match(/^\w+/)&.to_s
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# The root element as XML, with every element and attribute in no
|
|
92
|
+
# namespace (`xml:` attributes keep their prefix). The lutaml-model
|
|
93
|
+
# mappings are namespace-free: they skip a prefixed element, and a
|
|
94
|
+
# nested `relation/bibitem` in a default namespace raises in
|
|
95
|
+
# lutaml-model 0.8.175. Moxml cannot drop a namespace in place, so
|
|
96
|
+
# this builds a copy. (Mirror of relaton cli.rb without_namespaces.)
|
|
97
|
+
def without_namespaces(doc)
|
|
98
|
+
plain = Moxml.new.create_document
|
|
99
|
+
plain.add_child plain_element(plain, doc.root)
|
|
100
|
+
plain.root.to_xml
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def plain_element(doc, src)
|
|
104
|
+
elm = doc.create_element(src.name)
|
|
105
|
+
# An attribute in no namespace wins over a prefixed one of the
|
|
106
|
+
# same local name (`type` over `a:type`).
|
|
107
|
+
src.attributes.sort_by { |a| a.namespace ? 0 : 1 }.each do |attr|
|
|
108
|
+
elm[plain_name(attr)] = attr.value
|
|
109
|
+
end
|
|
110
|
+
src.children.each { |child| elm.add_child plain_node(doc, child) }
|
|
111
|
+
elm
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# The local name; an `xml:` attribute keeps its prefix.
|
|
115
|
+
def plain_name(attr)
|
|
116
|
+
attr.namespace&.prefix == "xml" ? "xml:#{attr.name}" : attr.name
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# An entity reference (` `) stays as its text: lutaml-model
|
|
120
|
+
# cannot read an entity it has no declaration for. A processing
|
|
121
|
+
# instruction becomes an empty text.
|
|
122
|
+
def plain_node(doc, src)
|
|
123
|
+
case src
|
|
124
|
+
when Moxml::Element then plain_element(doc, src)
|
|
125
|
+
when Moxml::Cdata then doc.create_cdata(src.content)
|
|
126
|
+
when Moxml::Text then doc.create_text(src.content)
|
|
127
|
+
when Moxml::Comment then doc.create_comment(src.content)
|
|
128
|
+
when Moxml::EntityReference then doc.create_text("&#{src.name};")
|
|
129
|
+
else doc.create_text("")
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
@@ -67,8 +67,9 @@ module Metanorma
|
|
|
67
67
|
|
|
68
68
|
def bibdata_from_xml(model, node)
|
|
69
69
|
node or return
|
|
70
|
-
|
|
71
|
-
|
|
70
|
+
native = node.adapter_node.native
|
|
71
|
+
force_primary_docidentifier_xml(native)
|
|
72
|
+
model.bibdata = BibdataParser.from_xml(native.to_xml)
|
|
72
73
|
end
|
|
73
74
|
|
|
74
75
|
def force_primary_docidentifier_xml(node)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
require "relaton"
|
|
2
2
|
require "relaton/bib"
|
|
3
|
-
|
|
3
|
+
require_relative "../config/bibdata_parser"
|
|
4
4
|
|
|
5
5
|
module Metanorma
|
|
6
6
|
class Collection
|
|
@@ -136,8 +136,9 @@ module Metanorma
|
|
|
136
136
|
from_xml (Nokogiri::XML(File.read(file, encoding: "UTF-8"),
|
|
137
137
|
&:huge))
|
|
138
138
|
when :yaml
|
|
139
|
-
|
|
140
|
-
|
|
139
|
+
Config::BibdataParser.from_yaml(
|
|
140
|
+
YAML.safe_load_file(file, permitted_classes: [Date, Symbol]),
|
|
141
|
+
)
|
|
141
142
|
end
|
|
142
143
|
end
|
|
143
144
|
end
|
|
@@ -3,20 +3,22 @@
|
|
|
3
3
|
module Metanorma
|
|
4
4
|
class Collection
|
|
5
5
|
class Renderer
|
|
6
|
-
def file_compile_formats(filename, identifier, opts)
|
|
6
|
+
def file_compile_formats(filename, identifier, opts, processor)
|
|
7
7
|
f = @files.get(identifier, :outputs)
|
|
8
8
|
format = opts[:extension_keys]
|
|
9
9
|
concatenate_presentation?({ format: }) and format << :presentation
|
|
10
10
|
format.each do |e|
|
|
11
11
|
e == :pdf and output_filename = opts[:pdffile]
|
|
12
|
-
file_compile_format(filename, identifier, e, f,
|
|
12
|
+
file_compile_format(filename, identifier, e, f,
|
|
13
|
+
new_output_fname: output_filename,
|
|
14
|
+
processor: processor)
|
|
13
15
|
end
|
|
14
16
|
@files.set(identifier, :outputs, f)
|
|
15
17
|
end
|
|
16
18
|
|
|
17
19
|
# Generate output filename with correct extension
|
|
18
|
-
def output_filename_with_extension(fname, format)
|
|
19
|
-
ext =
|
|
20
|
+
def output_filename_with_extension(fname, format, processor)
|
|
21
|
+
ext = processor.output_formats[format]
|
|
20
22
|
fname_base = File.basename(fname)
|
|
21
23
|
# ONLY replace .xml or .html extensions, NEVER remove section numbers
|
|
22
24
|
if /\.(xml|html)$/.match?(fname_base)
|
|
@@ -41,7 +43,7 @@ module Metanorma
|
|
|
41
43
|
|
|
42
44
|
# Apply custom filename pattern with substitutions
|
|
43
45
|
def apply_custom_filename_pattern(ident, custom_fname, output_fname,
|
|
44
|
-
|
|
46
|
+
ext)
|
|
45
47
|
idx = @files.get(ident, :idx)
|
|
46
48
|
original_file = @files.get(ident, :ref)
|
|
47
49
|
basename = File.basename(original_file, ".*")
|
|
@@ -53,30 +55,34 @@ format)
|
|
|
53
55
|
if File.dirname(custom_fname) == "."
|
|
54
56
|
output_fname
|
|
55
57
|
else
|
|
56
|
-
preserve_output_dir_structure(custom_fname, output_fname,
|
|
57
|
-
|
|
58
|
+
preserve_output_dir_structure(custom_fname, output_fname, ext,
|
|
59
|
+
explicit_custom: true)
|
|
58
60
|
end
|
|
59
61
|
end
|
|
60
62
|
|
|
61
63
|
# Apply directory preservation for files with directory structure
|
|
62
|
-
def apply_directory_preservation(fname, output_fname,
|
|
64
|
+
def apply_directory_preservation(fname, output_fname, ext)
|
|
63
65
|
fname_dir = File.dirname(fname)
|
|
64
66
|
output_fname = File.join(fname_dir, output_fname)
|
|
65
|
-
preserve_output_dir_structure(fname, output_fname,
|
|
67
|
+
preserve_output_dir_structure(fname, output_fname, ext)
|
|
66
68
|
end
|
|
67
69
|
|
|
68
|
-
# Compile single format for a file and update outputs hash
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
70
|
+
# Compile single format for a file and update outputs hash; opts
|
|
71
|
+
# carries the custom pdf filename (:new_output_fname) and the flavour
|
|
72
|
+
# :processor resolving the format-to-extension mapping.
|
|
73
|
+
def file_compile_format(fname, ident, format, outputs, opts)
|
|
74
|
+
ext = opts[:processor].output_formats[format]
|
|
75
|
+
output_fname = output_filename_with_extension(fname, format,
|
|
76
|
+
opts[:processor])
|
|
72
77
|
output_fname = handle_new_output_filename(output_fname,
|
|
73
|
-
new_output_fname)
|
|
74
|
-
if !new_output_fname &&
|
|
78
|
+
opts[:new_output_fname])
|
|
79
|
+
if !opts[:new_output_fname] &&
|
|
80
|
+
(custom = @files.preserve_directory_structure?(ident))
|
|
75
81
|
output_fname = apply_custom_filename_pattern(ident, custom,
|
|
76
|
-
output_fname,
|
|
77
|
-
elsif !new_output_fname && File.dirname(fname) != "."
|
|
82
|
+
output_fname, ext)
|
|
83
|
+
elsif !opts[:new_output_fname] && File.dirname(fname) != "."
|
|
78
84
|
output_fname = apply_directory_preservation(fname, output_fname,
|
|
79
|
-
|
|
85
|
+
ext)
|
|
80
86
|
end
|
|
81
87
|
should_skip = /html$/.match?(ext) && @files.get(ident, :sectionsplit)
|
|
82
88
|
should_skip or outputs[format] = File.join(@outdir, output_fname)
|
|
@@ -126,14 +132,14 @@ format)
|
|
|
126
132
|
"#{custom_basename}#{ext}"
|
|
127
133
|
end
|
|
128
134
|
|
|
129
|
-
# Preserve directory structure from input filename in output
|
|
130
|
-
|
|
135
|
+
# Preserve directory structure from input filename in output; ext is
|
|
136
|
+
# the output extension string the caller resolved for its format.
|
|
137
|
+
def preserve_output_dir_structure(fname, output_fname, ext = "",
|
|
131
138
|
explicit_custom: false)
|
|
132
139
|
fname_dir = File.dirname(fname)
|
|
133
140
|
return output_fname if fname_dir == "."
|
|
134
141
|
|
|
135
142
|
output_basename = File.basename(output_fname)
|
|
136
|
-
ext = format ? @compile.processor.output_formats[format].to_s : ""
|
|
137
143
|
should_move = should_move_to_subdirectory?(ext, output_basename,
|
|
138
144
|
explicit_custom)
|
|
139
145
|
if should_move
|
|
@@ -29,8 +29,11 @@ module Metanorma
|
|
|
29
29
|
Metanorma::Collection::XrefProcess::xref_process(xml, xml, nil, docid,
|
|
30
30
|
@isodoc, sso)
|
|
31
31
|
# post (semantic only): intra-doc xref/eref turned into internal erefs;
|
|
32
|
-
# repo bibitems copied; indirect bibliography inserted.
|
|
33
|
-
|
|
32
|
+
# repo bibitems copied; indirect bibliography inserted. The
|
|
33
|
+
# indirect-reference state is a per-member memo (not renderer
|
|
34
|
+
# instance state): members run concurrently on the render pool, and
|
|
35
|
+
# a shared memo would let one member's reset drop another member's
|
|
36
|
+
# already-updated-anchor guards mid-pass.
|
|
34
37
|
@nested or update_indirect_refs_to_docs(xml, docid, internal_refs, sso)
|
|
35
38
|
# post: bibitem[@type='internal'] anchors resolved to the containing
|
|
36
39
|
# collection document.
|
|
@@ -191,58 +194,72 @@ presxml)
|
|
|
191
194
|
# Resolve erefs to a container of ids in another doc,
|
|
192
195
|
# to an anchor eref (direct link)
|
|
193
196
|
def update_indirect_refs_to_docs(docxml, _docid, internal_refs, presxml)
|
|
194
|
-
|
|
195
|
-
presxml)
|
|
197
|
+
f, memo = update_indirect_refs_prep(docxml, presxml)
|
|
196
198
|
internal_refs.each do |schema, ids|
|
|
197
|
-
add_suffix = doc_suffix && doc_type &&
|
|
199
|
+
add_suffix = memo[:doc_suffix] && memo[:doc_type] &&
|
|
200
|
+
memo[:doc_type] != schema
|
|
198
201
|
ids.each do |id, file|
|
|
199
202
|
f[file] ||= { url: @files.url?(file),
|
|
200
203
|
parentid: @files.get(file) && @files.get(file,
|
|
201
204
|
:parentid) }
|
|
202
|
-
k = indirect_ref_key(schema, id,
|
|
203
|
-
update_indirect_refs_to_docs1(f[file], k, file,
|
|
205
|
+
k = indirect_ref_key(memo, schema, id, add_suffix)
|
|
206
|
+
update_indirect_refs_to_docs1(f[file], k, file, memo)
|
|
204
207
|
end
|
|
205
208
|
end
|
|
206
209
|
end
|
|
207
210
|
|
|
211
|
+
# Prepares the per-member indirect-reference pass: the file container
|
|
212
|
+
# plus a memo hash carrying the pre-gathered bibitems and erefs, the
|
|
213
|
+
# document suffix/type, the schema/id key cache, and the
|
|
214
|
+
# already-updated-anchor guards. The memo is a local rather than
|
|
215
|
+
# renderer instance state, so members on the render pool cannot reset
|
|
216
|
+
# each other's state mid-pass.
|
|
208
217
|
def update_indirect_refs_prep(docxml, presxml)
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
218
|
+
memo = { updated_anchors: {}, indirect_keys: {},
|
|
219
|
+
bibitems: Util::gather_bibitems(docxml),
|
|
220
|
+
erefs: Util::gather_bibitemids(docxml, presxml),
|
|
221
|
+
doc_suffix: docxml.root["document_suffix"],
|
|
222
|
+
doc_type: docxml.root["type"] }
|
|
223
|
+
[{}, memo]
|
|
213
224
|
end
|
|
214
225
|
|
|
215
|
-
def indirect_ref_key(schema, id,
|
|
226
|
+
def indirect_ref_key(memo, schema, id, add_suffix)
|
|
216
227
|
/^#{schema}_/.match?(id) and return id
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
228
|
+
keys = memo[:indirect_keys]
|
|
229
|
+
x = keys.dig(schema, id) and return x
|
|
230
|
+
keys[schema] ||= {}
|
|
231
|
+
keys[schema][id] = if add_suffix
|
|
232
|
+
"#{schema}_#{id}_#{memo[:doc_suffix]}"
|
|
233
|
+
else
|
|
234
|
+
"#{schema}_#{id}"
|
|
235
|
+
end
|
|
224
236
|
end
|
|
225
237
|
|
|
226
|
-
def update_indirect_refs_to_docs1(filec, key, file,
|
|
227
|
-
erefs[key]&.each do |e|
|
|
238
|
+
def update_indirect_refs_to_docs1(filec, key, file, memo)
|
|
239
|
+
memo[:erefs][key]&.each do |e|
|
|
228
240
|
e["citeas"] = file
|
|
229
241
|
update_indirect_refs_to_docs_anchor(e, file, filec[:url],
|
|
230
|
-
filec[:parentid])
|
|
242
|
+
filec[:parentid], memo)
|
|
231
243
|
end
|
|
232
|
-
update_indirect_refs_to_docs_docid(bibitems[key], file)
|
|
244
|
+
update_indirect_refs_to_docs_docid(memo[:bibitems][key], file)
|
|
233
245
|
end
|
|
234
246
|
|
|
235
|
-
def update_indirect_refs_to_docs_anchor(eref, file, url, parentid)
|
|
247
|
+
def update_indirect_refs_to_docs_anchor(eref, file, url, parentid, memo)
|
|
236
248
|
a = eref.at(".//#{ANCHOR_XPATH}") or return
|
|
237
249
|
parentid and file = "#{parentid}_#{file}"
|
|
238
250
|
existing = a.text
|
|
239
251
|
anchor = if url then existing
|
|
240
|
-
else
|
|
241
|
-
@indirect_keys[existing] ||= {}
|
|
242
|
-
@indirect_keys[existing][file] ||= Metanorma::Utils::to_ncname("#{existing}_#{file}")
|
|
252
|
+
else indirect_ncname(memo, existing, file)
|
|
243
253
|
end
|
|
244
|
-
|
|
245
|
-
|
|
254
|
+
memo[:updated_anchors][existing] or a.children = anchor
|
|
255
|
+
memo[:updated_anchors][anchor] = true
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# The NCName anchor binding an eref's existing anchor text to its
|
|
259
|
+
# target file, memoised in the member pass's key cache.
|
|
260
|
+
def indirect_ncname(memo, existing, file)
|
|
261
|
+
keys = memo[:indirect_keys][existing] ||= {}
|
|
262
|
+
keys[file] ||= Metanorma::Utils::to_ncname("#{existing}_#{file}")
|
|
246
263
|
end
|
|
247
264
|
|
|
248
265
|
def update_indirect_refs_to_docs_docid(bib, file)
|
|
@@ -256,34 +273,28 @@ presxml)
|
|
|
256
273
|
def update_anchors(bib, docid, erefs, erefs_no_anchor, erefs_anchors)
|
|
257
274
|
@files.get(docid) or error_anchor(erefs, docid)
|
|
258
275
|
has_anchors, url, ncn_docid = update_anchors_prep(docid)
|
|
276
|
+
concat = {} # per-member memo, as with the indirect-reference pass
|
|
259
277
|
erefs_no_anchor.each do |e|
|
|
260
278
|
update_anchor_create_loc(bib, e, docid)
|
|
261
279
|
end
|
|
262
280
|
!url && has_anchors or return
|
|
263
281
|
erefs_anchors.each do |e|
|
|
264
|
-
update_anchors1(docid, ncn_docid, e)
|
|
282
|
+
update_anchors1(concat, docid, ncn_docid, e)
|
|
265
283
|
end
|
|
266
284
|
end
|
|
267
285
|
|
|
268
|
-
def update_anchors1(docid, ncn_docid, anchor)
|
|
269
|
-
|
|
270
|
-
if @files.get(docid).dig(:anchors_lookup,
|
|
271
|
-
anchor.content =
|
|
286
|
+
def update_anchors1(concat, docid, ncn_docid, anchor)
|
|
287
|
+
concat[anchor.text] ||= "#{ncn_docid}_#{anchor.text}"
|
|
288
|
+
if @files.get(docid).dig(:anchors_lookup, concat[anchor.text])
|
|
289
|
+
anchor.content = concat[anchor.text]
|
|
272
290
|
end
|
|
273
291
|
end
|
|
274
292
|
|
|
275
293
|
def update_anchors_prep(docid)
|
|
276
|
-
@concat_anchors = {}
|
|
277
294
|
[@files.get(docid)&.key?(:anchors_lookup), @files.url?(docid),
|
|
278
295
|
Metanorma::Utils::to_ncname(docid)]
|
|
279
296
|
end
|
|
280
297
|
|
|
281
|
-
# encode both prefix and suffix to NCName
|
|
282
|
-
def suffix_anchor_indirect(prefix, suffix)
|
|
283
|
-
k = "#{prefix}_#{suffix}"
|
|
284
|
-
@ncnames[k] ||= Metanorma::Utils::to_ncname(k)
|
|
285
|
-
end
|
|
286
|
-
|
|
287
298
|
# if there is a crossref to another document, with no anchor, retrieve the
|
|
288
299
|
# anchor given the locality, and insert it into the crossref
|
|
289
300
|
def update_anchor_create_loc(_bib, eref, docid)
|
|
@@ -2,6 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
require "isodoc"
|
|
4
4
|
require "metanorma-utils"
|
|
5
|
+
require "concurrent-ruby"
|
|
6
|
+
# eprun (via twitter_cldr, via isodoc) sets its active Unicode version as a
|
|
7
|
+
# thread-local at require time, on whichever thread first loads it. Load it
|
|
8
|
+
# here, before the member pool starts, so that require happens once on the
|
|
9
|
+
# main thread and the pool workers can then set their own thread-local
|
|
10
|
+
# (initialise_pool_thread) instead of racing over who first requires the
|
|
11
|
+
# gem mid-render.
|
|
12
|
+
require "eprun"
|
|
5
13
|
require_relative "fileparse"
|
|
6
14
|
require_relative "filelocation"
|
|
7
15
|
require_relative "../artifact_store"
|
|
@@ -10,37 +18,76 @@ require "json"
|
|
|
10
18
|
module Metanorma
|
|
11
19
|
class Collection
|
|
12
20
|
class Renderer
|
|
21
|
+
# One member's rendering failure, buffered until the member pool joins
|
|
22
|
+
# so that failures can be reported in manifest order regardless of
|
|
23
|
+
# completion order.
|
|
24
|
+
MemberFailure = Struct.new(:index, :identifier, :errors, :missing)
|
|
25
|
+
|
|
13
26
|
# compile and output individual file in collection
|
|
14
27
|
# warn "metanorma compile -x html #{f.path}"
|
|
15
|
-
def file_compile(file, filename, identifier)
|
|
28
|
+
def file_compile(file, filename, identifier, index)
|
|
16
29
|
sectionsplit_val = @files.get(identifier, :sectionsplit)
|
|
17
30
|
sectionsplit_val and return
|
|
18
31
|
opts = compile_options_base(identifier)
|
|
19
32
|
.merge(compile_options_update(identifier))
|
|
33
|
+
.merge(compile_options_bare_after_first(index))
|
|
20
34
|
|
|
21
|
-
|
|
22
|
-
@compile.compile file, opts
|
|
35
|
+
compile = member_compile(file, opts)
|
|
23
36
|
@files.set(identifier, :outputs, {})
|
|
24
|
-
file_compile_formats(filename, identifier, opts
|
|
25
|
-
|
|
37
|
+
file_compile_formats(filename, identifier, opts,
|
|
38
|
+
member_processor(compile))
|
|
39
|
+
file_compile_verify(identifier, index, compile.errors)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Run one member's conversion on its own Compile instance
|
|
43
|
+
# (metanorma#705): each member's error pool collects only that
|
|
44
|
+
# member's failures, so attribution is order-independent under the
|
|
45
|
+
# member pool. The previous design sliced a shared @compile.errors by
|
|
46
|
+
# array-position delta, which was only sound while members compiled
|
|
47
|
+
# sequentially. Construction cost is negligible (~0.1ms incl. flavour
|
|
48
|
+
# load) next to a member render; the renderer-level @compile remains
|
|
49
|
+
# for flavour loading and the collection-level font/PDF handling.
|
|
50
|
+
def member_compile(file, opts)
|
|
51
|
+
compile = Compile.new
|
|
52
|
+
compile.load_flavor(@flavor)
|
|
53
|
+
compile.compile file, opts
|
|
54
|
+
compile
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# The flavour processor for output-path mapping: a compile that
|
|
58
|
+
# returned early (unsupported extensions, unreadable input) never
|
|
59
|
+
# reached its processor; the registry resolves the same processor
|
|
60
|
+
# object the compile would, so path mapping stays deterministic and
|
|
61
|
+
# independent of member order.
|
|
62
|
+
def member_processor(compile)
|
|
63
|
+
compile.processor ||
|
|
64
|
+
@registry.find_processor(Util::taste2flavor(@flavor).to_sym)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# bare-after-first: the directive previously mutated the shared compile
|
|
68
|
+
# options as the sequential loop walked the keys; under the member pool
|
|
69
|
+
# it becomes a per-member option keyed by the member's position in the
|
|
70
|
+
# file lookup (the first key renders bare-free, all later ones bare).
|
|
71
|
+
def compile_options_bare_after_first(index)
|
|
72
|
+
bare = index.positive? &&
|
|
73
|
+
@directives.detect { |d| d.key == "bare-after-first" }
|
|
74
|
+
bare ? { bare: true } : {}
|
|
26
75
|
end
|
|
27
76
|
|
|
28
77
|
# A member whose rendering raised produces no output file, but the
|
|
29
78
|
# collection build used to continue, exit 0, and link the absent
|
|
30
79
|
# document from index.html -- silent data loss (#586). Two
|
|
31
|
-
# complementary checks close that hole. (1)
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
# errors -- sound while members compile sequentially (the current
|
|
37
|
-
# design; @compile's own format-level parallelism is joined before
|
|
38
|
-
# compile returns). (2) Independently of whether anything was
|
|
80
|
+
# complementary checks close that hole. (1) Each member compiles with
|
|
81
|
+
# its own Compile instance (#705), whose error pool therefore holds
|
|
82
|
+
# only this member's conversion failures -- attribution is independent
|
|
83
|
+
# of execution order (the format-level pool inside a compile joins
|
|
84
|
+
# before compile returns). (2) Independently of whether anything was
|
|
39
85
|
# pooled, verify the outputs registered for the member exist on
|
|
40
|
-
# disk. Failures are
|
|
41
|
-
# renders every remaining member before
|
|
42
|
-
# so one run reports all
|
|
43
|
-
|
|
86
|
+
# disk. Failures are buffered here and accumulated; the collection
|
|
87
|
+
# renders every remaining member before finalise_member_failures
|
|
88
|
+
# reports them and Collection#render aborts, so one run reports all
|
|
89
|
+
# failed members.
|
|
90
|
+
def file_compile_verify(identifier, index, errors)
|
|
44
91
|
# Section-split parts (marked with :parentid) register format
|
|
45
92
|
# slots they never produce -- their :xml slot in particular is
|
|
46
93
|
# never written, and under filename renaming it aliases to the
|
|
@@ -51,10 +98,32 @@ module Metanorma
|
|
|
51
98
|
missing = (@files.get(identifier, :outputs) || {})
|
|
52
99
|
.reject { |_fmt, path| File.exist?(path) }
|
|
53
100
|
errors.empty? && missing.empty? and return
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
101
|
+
record_member_failure(index, identifier, errors, missing)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Buffer a member failure under the mutex: pool workers complete out
|
|
105
|
+
# of manifest order, and appends from multiple threads are not a
|
|
106
|
+
# synchronisation point we rely on.
|
|
107
|
+
def record_member_failure(index, identifier, errors, missing)
|
|
108
|
+
@failures_mutex.synchronize do
|
|
109
|
+
@member_failures << MemberFailure.new(index, identifier, errors,
|
|
110
|
+
missing)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Flush the buffered member failures in manifest order: the error
|
|
115
|
+
# lines, the log entries (and so the .err.html artifact) and the
|
|
116
|
+
# fatal_errors list are ordered by the manifest, never by completion
|
|
117
|
+
# order. Runs before Collection#render aborts on fatal_errors (#586).
|
|
118
|
+
def finalise_member_failures
|
|
119
|
+
@member_failures.sort_by(&:index).each do |failure|
|
|
120
|
+
file_compile_failure_log(failure.identifier, failure.errors,
|
|
121
|
+
failure.missing)
|
|
122
|
+
msg = file_compile_failure_msg(failure.identifier, failure.errors,
|
|
123
|
+
failure.missing)
|
|
124
|
+
::Metanorma::Util.log("[metanorma] Error: #{msg}", :error)
|
|
125
|
+
fatal_errors << msg
|
|
126
|
+
end
|
|
58
127
|
end
|
|
59
128
|
|
|
60
129
|
def file_compile_failure_log(identifier, errors, missing)
|
|
@@ -110,14 +179,29 @@ module Metanorma
|
|
|
110
179
|
|
|
111
180
|
# process each file in the collection
|
|
112
181
|
# files are held in memory, and altered as postprocessing
|
|
182
|
+
#
|
|
183
|
+
# Members are rendered on a bounded pool (metanorma#705): members are
|
|
184
|
+
# independent render units, and the sequential design existed for
|
|
185
|
+
# error attribution, not data dependence. Shared-state audit:
|
|
186
|
+
# - FileLookup entries are keyed per member; the only entry a member
|
|
187
|
+
# writes is its own :outputs hash. Distinct keys, distinct hashes.
|
|
188
|
+
# - each member builds its own Compile instance, so no compile state
|
|
189
|
+
# (errors, processor, log) is shared between members;
|
|
190
|
+
# - the cross-reference passes mutate only the member's own XML tree
|
|
191
|
+
# and per-member memo structures (localised in fileparse.rb); the
|
|
192
|
+
# renderer-level memos the passes read (svg hotspot index) are pure
|
|
193
|
+
# functions of the file lookup, which is frozen once init completes;
|
|
194
|
+
# - attachment copies and the DisambigFiles nametable stay on the main
|
|
195
|
+
# thread;
|
|
196
|
+
# - failures are buffered under a mutex and flushed in manifest order
|
|
197
|
+
# once the pool joins, and the navigation/coverpage/concatenation
|
|
198
|
+
# passes are built from the manifest afterwards (navigation.rb), so
|
|
199
|
+
# completion order never leaks into the output.
|
|
113
200
|
def files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
114
201
|
warn "\n\n\n\n\nRender Files: #{DateTime.now.strftime('%H:%M:%S')}"
|
|
115
202
|
internal_refs = locate_internal_refs
|
|
203
|
+
jobs = []
|
|
116
204
|
@files.keys.each_with_index do |ident, i|
|
|
117
|
-
i.positive? && @directives.detect do |d|
|
|
118
|
-
d.key == "bare-after-first"
|
|
119
|
-
end and
|
|
120
|
-
@compile_options.merge!(bare: true)
|
|
121
205
|
if @files.get(ident, :attachment) then copy_file_to_dest(ident)
|
|
122
206
|
else
|
|
123
207
|
# Distributed reinflation: the manifest carries every member
|
|
@@ -126,24 +210,79 @@ module Metanorma
|
|
|
126
210
|
# other members' sections are rendered by sibling runs over
|
|
127
211
|
# the same store.
|
|
128
212
|
@only_members and !render_member?(ident) and next
|
|
129
|
-
|
|
130
|
-
@preserve_unresolved && member_stored?(ident, file) and next
|
|
131
|
-
warn "\n\n\n\n\nProcess #{fname}: #{DateTime.now.strftime('%H:%M:%S')}"
|
|
132
|
-
collection_xml = update_xrefs(file, ident, internal_refs)
|
|
133
|
-
@preserve_unresolved and
|
|
134
|
-
store_member_artefacts(ident, file, collection_xml)
|
|
135
|
-
# Strip .xml or .html extension, but NOT section numbers like .0
|
|
136
|
-
fname_base = File.basename(fname)
|
|
137
|
-
collection_filename = fname_base
|
|
138
|
-
/\.(xml|html)$/.match?(fname_base) and
|
|
139
|
-
collection_filename = fname_base.sub(/\.(xml|html)$/, "")
|
|
140
|
-
collection_xml_path = File.join(Dir.tmpdir,
|
|
141
|
-
"#{collection_filename}.xml")
|
|
142
|
-
File.write collection_xml_path, collection_xml, encoding: "UTF-8"
|
|
143
|
-
file_compile(collection_xml_path, fname, ident)
|
|
144
|
-
FileUtils.rm_f(collection_xml_path)
|
|
213
|
+
jobs << [i, ident]
|
|
145
214
|
end
|
|
146
215
|
end
|
|
216
|
+
render_members(jobs, internal_refs)
|
|
217
|
+
finalise_member_failures
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Render the selected members on a bounded pool. Default 4 workers,
|
|
221
|
+
# overridable with the :member_pool_size render option.
|
|
222
|
+
def render_members(jobs, internal_refs)
|
|
223
|
+
jobs.empty? and return
|
|
224
|
+
pool = Concurrent::FixedThreadPool.new(member_pool_size)
|
|
225
|
+
jobs.each do |(index, ident)|
|
|
226
|
+
pool.post { render_member(index, ident, internal_refs) }
|
|
227
|
+
end
|
|
228
|
+
pool.shutdown
|
|
229
|
+
pool.wait_for_termination
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# One member render unit: cross-reference passes over the member's own
|
|
233
|
+
# tree, then compile and output. A member whose processing raises is
|
|
234
|
+
# recorded as that member's failure -- a raising job must never take
|
|
235
|
+
# its worker thread down or abort the pool, so the remaining members
|
|
236
|
+
# still render and the run reports all failures (#586).
|
|
237
|
+
def render_member(index, ident, internal_refs)
|
|
238
|
+
initialise_pool_thread
|
|
239
|
+
file, fname = @files.targetfile_id(ident, read: true)
|
|
240
|
+
@preserve_unresolved && member_stored?(ident, file) and return
|
|
241
|
+
warn "\n\n\n\n\nProcess #{fname}: #{DateTime.now.strftime('%H:%M:%S')}"
|
|
242
|
+
xml = member_collection_xml(file, ident, internal_refs)
|
|
243
|
+
xml_path = write_member_collection_xml(fname, xml)
|
|
244
|
+
file_compile(xml_path, fname, ident, index)
|
|
245
|
+
FileUtils.rm_f(xml_path)
|
|
246
|
+
rescue StandardError => e
|
|
247
|
+
record_member_failure(index, ident, ["#{e.class}: #{e.message}"], {})
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# Cross-reference the member's tree against the collection; a staging
|
|
251
|
+
# build additionally persists the stub-bearing artefacts to the store.
|
|
252
|
+
def member_collection_xml(file, ident, internal_refs)
|
|
253
|
+
collection_xml = update_xrefs(file, ident, internal_refs)
|
|
254
|
+
@preserve_unresolved and
|
|
255
|
+
store_member_artefacts(ident, file, collection_xml)
|
|
256
|
+
collection_xml
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
# Write the member's cross-referenced XML to a temp file for the
|
|
260
|
+
# compile. The temp basename drives the compiled output filenames, so
|
|
261
|
+
# it stays the member's own (per-collection-disambiguated) basename --
|
|
262
|
+
# distinct across members, so pool workers never share a temp path.
|
|
263
|
+
def write_member_collection_xml(fname, collection_xml)
|
|
264
|
+
# Strip .xml or .html extension, but NOT section numbers like .0
|
|
265
|
+
fname_base = File.basename(fname)
|
|
266
|
+
collection_filename = fname_base
|
|
267
|
+
/\.(xml|html)$/.match?(fname_base) and
|
|
268
|
+
collection_filename = fname_base.sub(/\.(xml|html)$/, "")
|
|
269
|
+
collection_xml_path = File.join(Dir.tmpdir,
|
|
270
|
+
"#{collection_filename}.xml")
|
|
271
|
+
File.write(collection_xml_path, collection_xml, encoding: "UTF-8")
|
|
272
|
+
collection_xml_path
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# Re-establish the per-thread gem defaults a fresh worker thread
|
|
276
|
+
# lacks. eprun keeps the active Unicode version in a thread-local
|
|
277
|
+
# that require time sets only on the loading thread (eprun.rb's
|
|
278
|
+
# `Eprun.current_version = Eprun::UcdVersion.latest`); on any other
|
|
279
|
+
# thread it reads nil and isodoc's index collation crashes with
|
|
280
|
+
# NoMethodError inside Eprun::UcdVersion.get. The gem itself is
|
|
281
|
+
# required up front (see the require block above), so every worker
|
|
282
|
+
# starts its job by giving its thread the same latest version the
|
|
283
|
+
# loading thread received.
|
|
284
|
+
def initialise_pool_thread
|
|
285
|
+
Eprun.current_version ||= Eprun::UcdVersion.latest
|
|
147
286
|
end
|
|
148
287
|
|
|
149
288
|
# Persist an isolated member's stub-bearing Semantic XML and its anchor
|
|
@@ -23,6 +23,9 @@ module Metanorma
|
|
|
23
23
|
builder.ul { |b| docrefs(mnf, b) }
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
+
# A document entry can carry nested entries of its own (publication >
|
|
27
|
+
# part > section, metanorma-cli#496); the block, if given, renders
|
|
28
|
+
# them inside the document's list item.
|
|
26
29
|
def docrefs(mnf, builder)
|
|
27
30
|
ident = docref_ident(mnf)
|
|
28
31
|
builder.li do |li|
|
|
@@ -31,6 +34,7 @@ module Metanorma
|
|
|
31
34
|
/[<>&]/.match?(x) ? x : @c.encode(x, :hexadecimal)
|
|
32
35
|
end.join
|
|
33
36
|
end
|
|
37
|
+
yield li if block_given?
|
|
34
38
|
end
|
|
35
39
|
end
|
|
36
40
|
|
|
@@ -39,6 +43,17 @@ module Metanorma
|
|
|
39
43
|
@c.decode(@isodoc.docid_prefix(nil, ident))
|
|
40
44
|
end
|
|
41
45
|
|
|
46
|
+
# nested entries of a document entry, as a sublist of its list item;
|
|
47
|
+
# all: list them regardless of their index flag
|
|
48
|
+
def nested_docrefs(mnf, builder, all: true)
|
|
49
|
+
kids = Array(mnf.entry)
|
|
50
|
+
all or kids = kids.select { |e| index?(e) }
|
|
51
|
+
kids.empty? and return
|
|
52
|
+
builder.ul do |u|
|
|
53
|
+
kids.each { |e| u << indexfile1(e, all:) }
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
42
57
|
# Check if file has a recognized MIME type (other than XML)
|
|
43
58
|
# If so, don't append .html (e.g., .svg, .png, .jpg, etc.)
|
|
44
59
|
def index_link(docref, ident)
|
|
@@ -72,19 +87,23 @@ module Metanorma
|
|
|
72
87
|
mnf.entry&.detect { |e| index?(e) }
|
|
73
88
|
end
|
|
74
89
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
90
|
+
# all: list the entry regardless of its index flag
|
|
91
|
+
def indexfile1(mnf, all: false)
|
|
92
|
+
all || index?(mnf) or return ""
|
|
93
|
+
cleanup_indexfile1(build_indexfile1(mnf, all:))
|
|
78
94
|
end
|
|
79
95
|
|
|
80
|
-
def build_indexfile1(mnf)
|
|
96
|
+
def build_indexfile1(mnf, all: false)
|
|
81
97
|
Nokogiri::HTML::Builder.new do |b|
|
|
82
|
-
if mnf.file
|
|
98
|
+
if mnf.file
|
|
99
|
+
docrefs(mnf, b) do |li|
|
|
100
|
+
nested_docrefs(mnf, li, all:)
|
|
101
|
+
end
|
|
83
102
|
else
|
|
84
103
|
b.li do |l|
|
|
85
104
|
l << indexfile_title(mnf)
|
|
86
105
|
l.ul do |u|
|
|
87
|
-
Array(mnf.entry).each { |e| u << indexfile1(e) }
|
|
106
|
+
Array(mnf.entry).each { |e| u << indexfile1(e, all:) }
|
|
88
107
|
end
|
|
89
108
|
end
|
|
90
109
|
end
|
|
@@ -125,7 +144,7 @@ module Metanorma
|
|
|
125
144
|
files.empty? and return nil
|
|
126
145
|
r = Nokogiri::HTML::Builder.new do |b|
|
|
127
146
|
b.ul do |u|
|
|
128
|
-
files.each { |f| docrefs(f, u) }
|
|
147
|
+
files.each { |f| docrefs(f, u) { |li| nested_docrefs(f, li) } }
|
|
129
148
|
end
|
|
130
149
|
end
|
|
131
150
|
r.doc.root&.to_html&.tr("\n", " ")
|
|
@@ -137,9 +156,9 @@ module Metanorma
|
|
|
137
156
|
ident = @c.decode(@isodoc.docid_prefix(nil, d.identifier.dup))
|
|
138
157
|
m << { "identifier" => ident, "file" => index_link(d, ident),
|
|
139
158
|
"title" => indexfile_title(d), "level" => d.type }
|
|
140
|
-
else
|
|
141
|
-
liquid_docrefs(d.entry).each { |m1| m << m1 }
|
|
142
159
|
end
|
|
160
|
+
# entries nested under a document follow it (metanorma-cli#496)
|
|
161
|
+
liquid_docrefs(d.entry).each { |m1| m << m1 }
|
|
143
162
|
end
|
|
144
163
|
end
|
|
145
164
|
end
|
|
@@ -17,6 +17,9 @@ module Metanorma
|
|
|
17
17
|
class Renderer
|
|
18
18
|
FORMATS = %i[html xml doc pdf pdf-portfolio].freeze
|
|
19
19
|
|
|
20
|
+
# Worker count for the member render pool (metanorma#705).
|
|
21
|
+
DEFAULT_MEMBER_POOL_SIZE = 4
|
|
22
|
+
|
|
20
23
|
attr_accessor :isodoc, :isodoc_presxml
|
|
21
24
|
attr_reader :xml, :compile, :compile_options, :documents, :outdir,
|
|
22
25
|
:manifest, :fatal_errors
|
|
@@ -94,9 +97,12 @@ module Metanorma
|
|
|
94
97
|
@c = HTMLEntities.new
|
|
95
98
|
@files_to_delete = []
|
|
96
99
|
# Per-member rendering failures (missing outputs, pooled compile
|
|
97
|
-
# errors),
|
|
98
|
-
#
|
|
100
|
+
# errors, raised exceptions), buffered by file_compile_verify and
|
|
101
|
+
# flushed in manifest order once the member pool joins; Collection
|
|
102
|
+
# #render aborts on them after every member has rendered (#586).
|
|
99
103
|
@fatal_errors = []
|
|
104
|
+
@failures_mutex = Mutex.new
|
|
105
|
+
@member_failures = []
|
|
100
106
|
@nested = options[:nested]
|
|
101
107
|
# if false, this is the root instance of Renderer
|
|
102
108
|
# if true, then this is not the last time Renderer will be run
|
|
@@ -130,6 +136,8 @@ module Metanorma
|
|
|
130
136
|
# pruning superseded content-hash versions on write; keep_cache retains
|
|
131
137
|
# every version (history / multiple states side by side).
|
|
132
138
|
@keep_cache = options[:keep_cache]
|
|
139
|
+
@member_pool_size = options[:member_pool_size] ||
|
|
140
|
+
DEFAULT_MEMBER_POOL_SIZE
|
|
133
141
|
|
|
134
142
|
@coverpage = options[:coverpage] || collection.coverpage
|
|
135
143
|
@coverpage_pdf_portflio = options[:coverpage_pdf_portfolio] ||
|
|
@@ -157,6 +165,11 @@ module Metanorma
|
|
|
157
165
|
create_non_existing_directory(@outdir)
|
|
158
166
|
end
|
|
159
167
|
|
|
168
|
+
# Worker count for the member render pool (see DEFAULT_MEMBER_POOL_SIZE).
|
|
169
|
+
def member_pool_size
|
|
170
|
+
@member_pool_size
|
|
171
|
+
end
|
|
172
|
+
|
|
160
173
|
def directives_normalise(directives)
|
|
161
174
|
@coverpage_pdf_portflio and
|
|
162
175
|
directives = directives_normalise_coverpage_pdf_portfolio(directives)
|
|
@@ -416,15 +416,35 @@ module Metanorma
|
|
|
416
416
|
# name (geometry_schema.cartesian_point_ISO_10303-42_2025_...), so
|
|
417
417
|
# the plain name resolves by prefix. Multiple equal-length matches
|
|
418
418
|
# are ambiguous and refuse, as do differing ids at the same key.
|
|
419
|
+
# Memoized per prefix: the index is immutable once built, and a
|
|
420
|
+
# mega-member's svgmap erefs hit this per stage pass (the linear
|
|
421
|
+
# scan showed up at 24%+ of a 104-style render's wall).
|
|
419
422
|
def hotspot_prefix_lookup(idx, prefix)
|
|
420
|
-
|
|
423
|
+
@hotspot_prefix_memo ||= {}
|
|
424
|
+
@hotspot_prefix_memo.fetch(prefix) do
|
|
425
|
+
@hotspot_prefix_memo[prefix] =
|
|
426
|
+
idx[prefix] || prefix_lookup_scan(idx, prefix)
|
|
427
|
+
end
|
|
428
|
+
end
|
|
421
429
|
|
|
422
|
-
|
|
430
|
+
# Candidate keys sharing the prefix, by binary search over the
|
|
431
|
+
# sorted index keys: keys with the stem prefix are contiguous in
|
|
432
|
+
# sort order, so the slice between the lower bound and the first
|
|
433
|
+
# non-matching key is the entire candidate set.
|
|
434
|
+
def prefix_lookup_scan(idx, prefix)
|
|
435
|
+
stem = "#{prefix}_"
|
|
436
|
+
keys = hotspot_sorted_keys(idx)
|
|
437
|
+
lo = keys.bsearch_index { |k| k >= stem } or return nil
|
|
438
|
+
candidates = keys[lo..].take_while { |k| k.start_with?(stem) }
|
|
423
439
|
return nil if candidates.empty?
|
|
424
440
|
|
|
425
|
-
shortest = candidates.min_by
|
|
426
|
-
ties = candidates.count { |k
|
|
427
|
-
ties == 1 ? shortest
|
|
441
|
+
shortest = candidates.min_by(&:length)
|
|
442
|
+
ties = candidates.count { |k| k.length == shortest.length }
|
|
443
|
+
ties == 1 ? idx[shortest] : nil
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
def hotspot_sorted_keys(idx)
|
|
447
|
+
@hotspot_sorted_keys ||= idx.keys.select { |k| idx[k] }.sort
|
|
428
448
|
end
|
|
429
449
|
end
|
|
430
450
|
end
|
|
@@ -54,7 +54,10 @@ module Metanorma
|
|
|
54
54
|
},
|
|
55
55
|
manifest: {
|
|
56
56
|
level: "collection", title: "Collection",
|
|
57
|
-
|
|
57
|
+
# [order, idx]: tasks complete out of order under the section
|
|
58
|
+
# split pool, and Ruby sorts are not guaranteed stable, so the
|
|
59
|
+
# split idx breaks displayorder ties deterministically.
|
|
60
|
+
docref: files.sort_by { |f| [f[:order], f[:idx]] }.each.map do |f|
|
|
58
61
|
# XML files are always stored flat, so fileref is always the basename
|
|
59
62
|
# f[:url] may include directory for HTML output, but XML is basename only
|
|
60
63
|
fileref = File.basename(f[:url])
|
|
@@ -67,8 +67,13 @@ module Metanorma
|
|
|
67
67
|
empty = empty_doc(xml)
|
|
68
68
|
empty1 = empty_attachments(empty)
|
|
69
69
|
@mutex = Mutex.new
|
|
70
|
-
#
|
|
71
|
-
|
|
70
|
+
# Per-section work mutates only its own xml.dup and the per-section
|
|
71
|
+
# hidden-ref container; the shared document is read-only once
|
|
72
|
+
# xref_preprocess has run serially above, and the manifest consumer
|
|
73
|
+
# re-sorts the results by [displayorder, split idx], so completion
|
|
74
|
+
# order is unobservable. Throttled to 1 during the isodoc#641
|
|
75
|
+
# fmt-xref refactor (92bd93a) and never restored. metanorma#703.
|
|
76
|
+
@pool = Concurrent::FixedThreadPool.new(4)
|
|
72
77
|
sectionsplit1(xml, empty, empty1, 0)
|
|
73
78
|
end
|
|
74
79
|
|
|
@@ -96,7 +101,7 @@ module Metanorma
|
|
|
96
101
|
)
|
|
97
102
|
warn "Sectionsplit: #{output_filename}"
|
|
98
103
|
a = sectionfile(xml, empty, output_filename, chunks,
|
|
99
|
-
parentnode)
|
|
104
|
+
parentnode).merge(idx: opt[:idx])
|
|
100
105
|
@mutex.synchronize { opt[:acc] << a }
|
|
101
106
|
end
|
|
102
107
|
end
|
data/lib/metanorma/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: metanorma
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.5.
|
|
4
|
+
version: 2.5.25
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-11-
|
|
11
|
+
date: 2026-11-19 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: asciidoctor
|
|
@@ -259,6 +259,7 @@ files:
|
|
|
259
259
|
- lib/metanorma/collection/artifact_store.rb
|
|
260
260
|
- lib/metanorma/collection/collection.rb
|
|
261
261
|
- lib/metanorma/collection/config/bibdata.rb
|
|
262
|
+
- lib/metanorma/collection/config/bibdata_parser.rb
|
|
262
263
|
- lib/metanorma/collection/config/compile_options.rb
|
|
263
264
|
- lib/metanorma/collection/config/config.rb
|
|
264
265
|
- lib/metanorma/collection/config/converters.rb
|