suma 0.5.41 → 0.5.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +4 -0
- data/lib/suma/cli/build.rb +5 -3
- data/lib/suma/cli/delivery.rb +61 -16
- data/lib/suma/delivery/expectations.rb +32 -3
- data/lib/suma/delivery/page_inventory.rb +66 -0
- data/lib/suma/delivery/site_verifier.rb +145 -1
- data/lib/suma/delivery.rb +1 -0
- data/lib/suma/schema_exporter.rb +5 -1
- data/lib/suma/schema_template/document.rb +4 -4
- data/lib/suma/schema_template/plain.rb +4 -4
- data/lib/suma/term_extractor.rb +23 -44
- data/lib/suma/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b19a3323b2babf94e043a9d44b58552d30a8216edce9a7b9481d4038114b2483
|
|
4
|
+
data.tar.gz: 56217001687acfb6b8aab49274e78dac63014336d42929f01f8ac007c387777d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5f5ce333acd1281bb2e708ee025649f448b210f1c0ec60a9c4856cb40a991a4ddc7c67b5d17bd5ece3faf00f98ce8421d4c5e585a1f179bae6c8f1a490516468
|
|
7
|
+
data.tar.gz: c25d489df407bce0993264896ed6c4eb8673b141ed586cdc03df428abb8009334b5bfbbc692bafd5969916d84b8f6a6e00e5ad4b1b9c88b678df199ec66af09d
|
data/.rubocop_todo.yml
CHANGED
|
@@ -277,6 +277,7 @@ RSpec/ContextWording:
|
|
|
277
277
|
RSpec/ExampleLength:
|
|
278
278
|
Exclude:
|
|
279
279
|
- 'spec/suma/delivery/expectations_spec.rb'
|
|
280
|
+
- 'spec/suma/delivery/page_inventory_spec.rb'
|
|
280
281
|
- 'spec/suma/anchor_validator_spec.rb'
|
|
281
282
|
- 'spec/suma/artifact_server_spec.rb'
|
|
282
283
|
- 'spec/suma/build_plan_spec.rb'
|
|
@@ -284,6 +285,7 @@ RSpec/ExampleLength:
|
|
|
284
285
|
- 'spec/suma/change_impact_spec.rb'
|
|
285
286
|
- 'spec/suma/change_manifest_spec.rb'
|
|
286
287
|
- 'spec/suma/cli/build_integration_spec.rb'
|
|
288
|
+
- 'spec/suma/cli/delivery_spec.rb'
|
|
287
289
|
- 'spec/suma/cli/export_spec.rb'
|
|
288
290
|
- 'spec/suma/cli/extract_terms_spec.rb'
|
|
289
291
|
- 'spec/suma/cli/generate_register_spec.rb'
|
|
@@ -377,12 +379,14 @@ RSpec/MessageSpies:
|
|
|
377
379
|
RSpec/MultipleExpectations:
|
|
378
380
|
Exclude:
|
|
379
381
|
- 'spec/suma/delivery/expectations_spec.rb'
|
|
382
|
+
- 'spec/suma/delivery/page_inventory_spec.rb'
|
|
380
383
|
- 'spec/suma/artifact_server_spec.rb'
|
|
381
384
|
- 'spec/suma/change_classification_spec.rb'
|
|
382
385
|
- 'spec/suma/change_impact_spec.rb'
|
|
383
386
|
- 'spec/suma/change_manifest_spec.rb'
|
|
384
387
|
- 'spec/suma/cli/build_integration_spec.rb'
|
|
385
388
|
- 'spec/suma/cli/build_spec.rb'
|
|
389
|
+
- 'spec/suma/cli/delivery_spec.rb'
|
|
386
390
|
- 'spec/suma/cli/export_spec.rb'
|
|
387
391
|
- 'spec/suma/cli/expressir_spec.rb'
|
|
388
392
|
- 'spec/suma/cli/extract_terms_spec.rb'
|
data/lib/suma/cli/build.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "etc"
|
|
3
4
|
require "thor"
|
|
4
5
|
|
|
5
6
|
module Suma
|
|
@@ -55,9 +56,10 @@ module Suma
|
|
|
55
56
|
"to plain_schemas/.suma-build-state.json)"
|
|
56
57
|
option :state_file, type: :string,
|
|
57
58
|
desc: "Path to the incremental build state file"
|
|
58
|
-
option :jobs, type: :numeric, default:
|
|
59
|
-
desc: "Concurrent schema export workers (
|
|
60
|
-
"
|
|
59
|
+
option :jobs, type: :numeric, default: Etc.nprocessors,
|
|
60
|
+
desc: "Concurrent schema export workers (the parser " \
|
|
61
|
+
"releases the GVL, so this is true parallelism; " \
|
|
62
|
+
"document compilation remains sequential)"
|
|
61
63
|
option :provenance, type: :string,
|
|
62
64
|
desc: "Write a provenance manifest (sha256 of inputs " \
|
|
63
65
|
"and outputs + toolchain versions) to this path"
|
data/lib/suma/cli/delivery.rb
CHANGED
|
@@ -17,7 +17,10 @@ module Suma
|
|
|
17
17
|
"hrefs to files, fragments to anchors in their target. Without " \
|
|
18
18
|
"--expectations the gate is zero broken links; with one, the " \
|
|
19
19
|
"manifest's thresholds (broken_links_max included) govern. " \
|
|
20
|
-
"
|
|
20
|
+
"Also censuses the page shape: build-directory paths, " \
|
|
21
|
+
"incomplete listings, unpublished schema remarks, asciidoc " \
|
|
22
|
+
"leaks, and the per-page img/link inventory. Intra-file " \
|
|
23
|
+
"fragment hrefs are reported but do not fail."
|
|
21
24
|
option :out, type: :string,
|
|
22
25
|
desc: "Also write the report as JSON to this path"
|
|
23
26
|
option :no_fragments, type: :boolean, default: false,
|
|
@@ -29,22 +32,35 @@ module Suma
|
|
|
29
32
|
"missing_express_max, broken_links_max, " \
|
|
30
33
|
"diagnostics_max: category => rule => " \
|
|
31
34
|
"max rows over the site's .err.html " \
|
|
32
|
-
"files
|
|
35
|
+
"files, forbidden_path_prefixes, " \
|
|
36
|
+
"build_dir_references_max, " \
|
|
37
|
+
"incomplete_listings_max, " \
|
|
38
|
+
"missing_schema_remarks_max, " \
|
|
39
|
+
"asciidoc_leaks_max); violations fail " \
|
|
40
|
+
"the verify"
|
|
41
|
+
option :inventory_baseline, type: :string,
|
|
42
|
+
desc: "YAML ratchet baseline (page => " \
|
|
43
|
+
"{imgs:, links:}) over the " \
|
|
44
|
+
"per-page inventory; pages whose " \
|
|
45
|
+
"counts changed fail the verify"
|
|
33
46
|
|
|
34
47
|
def verify(site_dir)
|
|
35
|
-
|
|
36
|
-
|
|
48
|
+
expectations = load_expectations
|
|
49
|
+
result = Suma::Delivery::SiteVerifier
|
|
50
|
+
.new(site_dir, **verify_options(expectations)).verify
|
|
37
51
|
log_verify(result)
|
|
38
|
-
failures
|
|
39
|
-
verify_failures(result, failures)
|
|
52
|
+
verify_failures(result, failures(result, expectations))
|
|
40
53
|
end
|
|
41
54
|
|
|
42
55
|
# Without an expectations manifest the contract is zero broken
|
|
43
56
|
# links. With one, broken_links_max IS the broken-link contract:
|
|
44
57
|
# the unconditional raise would fire first and make the cap
|
|
45
|
-
# unreachable (TRThurman, iso-10303#835).
|
|
46
|
-
#
|
|
47
|
-
#
|
|
58
|
+
# unreachable (TRThurman, iso-10303#835). The inventory baseline,
|
|
59
|
+
# when given, is its own gate: any page whose img/link counts
|
|
60
|
+
# moved since the baseline fails, whatever the expectations mode.
|
|
61
|
+
# Defined inside no_commands so Thor does not register a command
|
|
62
|
+
# for a helper and warn about the missing usage/description on
|
|
63
|
+
# every run.
|
|
48
64
|
no_commands do
|
|
49
65
|
def verify_failures(result, failures)
|
|
50
66
|
if options[:expectations]
|
|
@@ -55,6 +71,24 @@ module Suma
|
|
|
55
71
|
else
|
|
56
72
|
raise "broken links found (#{result.broken_count})"
|
|
57
73
|
end
|
|
74
|
+
raise_inventory_drift(result)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def raise_inventory_drift(result)
|
|
78
|
+
return unless options[:inventory_baseline]
|
|
79
|
+
|
|
80
|
+
changes = Suma::Delivery::PageInventory.changes(
|
|
81
|
+
result.report["page_inventory"], load_baseline
|
|
82
|
+
)
|
|
83
|
+
return if changes.empty?
|
|
84
|
+
|
|
85
|
+
raise "page inventory changed: " \
|
|
86
|
+
"#{Suma::Delivery::PageInventory.format_changes(changes)}"
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def load_baseline
|
|
90
|
+
Suma::Delivery::PageInventory
|
|
91
|
+
.load_baseline(File.read(options[:inventory_baseline]))
|
|
58
92
|
end
|
|
59
93
|
end
|
|
60
94
|
|
|
@@ -149,8 +183,9 @@ module Suma
|
|
|
149
183
|
|
|
150
184
|
private
|
|
151
185
|
|
|
152
|
-
def verify_options
|
|
153
|
-
{ check_fragments: !options[:no_fragments]
|
|
186
|
+
def verify_options(expectations)
|
|
187
|
+
{ check_fragments: !options[:no_fragments],
|
|
188
|
+
forbidden_path_prefixes: expectations&.forbidden_path_prefixes }
|
|
154
189
|
end
|
|
155
190
|
|
|
156
191
|
def load_expectations
|
|
@@ -160,10 +195,9 @@ module Suma
|
|
|
160
195
|
.from_yaml(File.read(options[:expectations]))
|
|
161
196
|
end
|
|
162
197
|
|
|
163
|
-
def
|
|
164
|
-
expectations
|
|
165
|
-
|
|
166
|
-
result.report["broken_links"])
|
|
198
|
+
def failures(result, expectations)
|
|
199
|
+
expectations&.violations(result.report,
|
|
200
|
+
result.report["broken_links"]) || []
|
|
167
201
|
end
|
|
168
202
|
|
|
169
203
|
def log_verify(result)
|
|
@@ -186,7 +220,18 @@ module Suma
|
|
|
186
220
|
"indexes=#{inv['index_pages']} " \
|
|
187
221
|
"attachments=#{inv['schema_attachments']} " \
|
|
188
222
|
"schema_docs=#{inv['schema_docs']}; " \
|
|
189
|
-
"missing-express pages=#{report['missing_express_pages'].size}"
|
|
223
|
+
"missing-express pages=#{report['missing_express_pages'].size}; " \
|
|
224
|
+
"#{page_shape_summary(report)}"
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# The page-shape census line (TRThurman, iso-10303#857):
|
|
228
|
+
# occurrences for build-dir paths and leaks, page counts for
|
|
229
|
+
# incomplete listings and unpublished remarks.
|
|
230
|
+
def page_shape_summary(report)
|
|
231
|
+
"build-dir=#{report['build_dir_occurrences']} " \
|
|
232
|
+
"listings=#{Array(report['incomplete_listings']).size} " \
|
|
233
|
+
"remarks=#{Array(report['missing_schema_remarks']).size} " \
|
|
234
|
+
"leaks=#{report['asciidoc_leak_occurrences']}"
|
|
190
235
|
end
|
|
191
236
|
|
|
192
237
|
# The err.html census the verify walked: total diagnostic rows over
|
|
@@ -17,6 +17,15 @@ module Suma
|
|
|
17
17
|
# identity of an err.html row is category + rule, not the code,
|
|
18
18
|
# which migrates across metanorma releases (iso-10303#859).
|
|
19
19
|
attribute :diagnostics_max, :hash
|
|
20
|
+
# Page-shape gate maxima over the verifier's page-shape census
|
|
21
|
+
# (TRThurman, iso-10303#857). The path prefixes scanned for
|
|
22
|
+
# build-directory leakage default to the verifier's list; set
|
|
23
|
+
# them here to override.
|
|
24
|
+
attribute :forbidden_path_prefixes, :string, collection: true
|
|
25
|
+
attribute :build_dir_references_max, :integer, default: -> { 0 }
|
|
26
|
+
attribute :incomplete_listings_max, :integer, default: -> { 0 }
|
|
27
|
+
attribute :missing_schema_remarks_max, :integer, default: -> { 0 }
|
|
28
|
+
attribute :asciidoc_leaks_max, :integer, default: -> { 0 }
|
|
20
29
|
|
|
21
30
|
key_value do
|
|
22
31
|
map "sections_min", to: :sections_min
|
|
@@ -26,6 +35,11 @@ module Suma
|
|
|
26
35
|
map "missing_express_max", to: :missing_express_max
|
|
27
36
|
map "broken_links_max", to: :broken_links_max
|
|
28
37
|
map "diagnostics_max", to: :diagnostics_max
|
|
38
|
+
map "forbidden_path_prefixes", to: :forbidden_path_prefixes
|
|
39
|
+
map "build_dir_references_max", to: :build_dir_references_max
|
|
40
|
+
map "incomplete_listings_max", to: :incomplete_listings_max
|
|
41
|
+
map "missing_schema_remarks_max", to: :missing_schema_remarks_max
|
|
42
|
+
map "asciidoc_leaks_max", to: :asciidoc_leaks_max
|
|
29
43
|
end
|
|
30
44
|
|
|
31
45
|
# Human-readable violation lines for every threshold the +report+
|
|
@@ -84,7 +98,8 @@ module Suma
|
|
|
84
98
|
|
|
85
99
|
# Every threshold's observed value, keyed by threshold name. The
|
|
86
100
|
# inventory carries the shape counts; missing_express_pages is a
|
|
87
|
-
# top-level report key
|
|
101
|
+
# top-level report key; the page-shape maxima count against the
|
|
102
|
+
# verifier's page-shape census.
|
|
88
103
|
def actuals(report, broken_links)
|
|
89
104
|
inventory = report["inventory"] || {}
|
|
90
105
|
{
|
|
@@ -92,9 +107,23 @@ module Suma
|
|
|
92
107
|
"index_pages" => inventory["index_pages"],
|
|
93
108
|
"schema_attachments_min" => inventory["schema_attachments"],
|
|
94
109
|
"schema_docs_min" => inventory["schema_docs"],
|
|
95
|
-
"missing_express_max" =>
|
|
96
|
-
Array(report["missing_express_pages"]).size,
|
|
110
|
+
"missing_express_max" => Array(report["missing_express_pages"]).size,
|
|
97
111
|
"broken_links_max" => broken_links.size,
|
|
112
|
+
**page_shape_actuals(report),
|
|
113
|
+
}
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# The page-shape census (TRThurman, iso-10303#857): occurrences
|
|
117
|
+
# for build-dir paths and asciidoc leaks, page counts for
|
|
118
|
+
# incomplete listings and unpublished remarks.
|
|
119
|
+
def page_shape_actuals(report)
|
|
120
|
+
{
|
|
121
|
+
"build_dir_references_max" => report["build_dir_occurrences"].to_i,
|
|
122
|
+
"incomplete_listings_max" =>
|
|
123
|
+
Array(report["incomplete_listings"]).size,
|
|
124
|
+
"missing_schema_remarks_max" =>
|
|
125
|
+
Array(report["missing_schema_remarks"]).size,
|
|
126
|
+
"asciidoc_leaks_max" => report["asciidoc_leak_occurrences"].to_i,
|
|
98
127
|
}
|
|
99
128
|
end
|
|
100
129
|
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "yaml"
|
|
4
|
+
|
|
5
|
+
module Suma
|
|
6
|
+
module Delivery
|
|
7
|
+
# Per-page inventory ratchet (TRThurman, iso-10303#857): the
|
|
8
|
+
# verifier's machine-readable page inventory (image and link counts
|
|
9
|
+
# per page) compared against a committed baseline, so silent
|
|
10
|
+
# rendering drift — a lost figure, a dropped navigation link —
|
|
11
|
+
# fails `delivery verify --inventory-baseline`. The baseline format
|
|
12
|
+
# is deliberately flat YAML, page path => { imgs:, links: }:
|
|
13
|
+
# free-form page keys, so it is plain data, not a fixed-shape
|
|
14
|
+
# lutaml-model document.
|
|
15
|
+
class PageInventory
|
|
16
|
+
# The counted quantities, in report-key order.
|
|
17
|
+
COUNT_KEYS = %w[imgs links].freeze
|
|
18
|
+
|
|
19
|
+
class << self
|
|
20
|
+
# The baseline census read from YAML: page path => count hash.
|
|
21
|
+
def load_baseline(yaml)
|
|
22
|
+
data = YAML.safe_load(yaml)
|
|
23
|
+
raise ArgumentError, "baseline must be a YAML mapping" unless
|
|
24
|
+
data.is_a?(Hash)
|
|
25
|
+
|
|
26
|
+
data.transform_values do |counts|
|
|
27
|
+
(counts || {}).transform_values(&:to_i)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Pages whose counts differ between the current inventory and
|
|
32
|
+
# the baseline: page path => { key => [baseline, current] }.
|
|
33
|
+
# Pages present on only one side count the missing side as zero.
|
|
34
|
+
def changes(current, baseline)
|
|
35
|
+
(current.keys | baseline.keys).filter_map do |page|
|
|
36
|
+
delta = count_changes(page, current, baseline)
|
|
37
|
+
next if delta.empty?
|
|
38
|
+
|
|
39
|
+
[page, delta]
|
|
40
|
+
end.to_h
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Human-readable drift lines: "page (imgs 2->1, links 5->4)".
|
|
44
|
+
def format_changes(changes)
|
|
45
|
+
changes.map do |page, delta|
|
|
46
|
+
moved = delta.map { |key, pair| "#{key} #{pair[0]}->#{pair[1]}" }
|
|
47
|
+
"#{page} (#{moved.join(', ')})"
|
|
48
|
+
end.join("; ")
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
# The moved counts of one page.
|
|
54
|
+
def count_changes(page, current, baseline)
|
|
55
|
+
COUNT_KEYS.filter_map do |key|
|
|
56
|
+
old = baseline.fetch(page, {})[key].to_i
|
|
57
|
+
now = current.fetch(page, {})[key].to_i
|
|
58
|
+
next if old == now
|
|
59
|
+
|
|
60
|
+
[key, [old, now]]
|
|
61
|
+
end.to_h
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "pathname"
|
|
5
|
+
require "expressir"
|
|
5
6
|
require "leptris"
|
|
6
7
|
|
|
7
8
|
module Suma
|
|
@@ -23,6 +24,28 @@ module Suma
|
|
|
23
24
|
# it cites an express reference the corpus never resolved.
|
|
24
25
|
MISSING_EXPRESS_MARKER = "Missing:express"
|
|
25
26
|
|
|
27
|
+
# Build-environment paths a published page must never name
|
|
28
|
+
# (TRThurman, iso-10303#857): paths pointing outside the site root
|
|
29
|
+
# have no meaning at delivery. Overridable per expectations
|
|
30
|
+
# through +forbidden_path_prefixes+.
|
|
31
|
+
DEFAULT_FORBIDDEN_PATH_PREFIXES =
|
|
32
|
+
%w[/home/ /Users/ /private/var/ /tmp/].freeze
|
|
33
|
+
|
|
34
|
+
# The whitespace-normalized fragment of a schema's first own
|
|
35
|
+
# remark that its schema-doc page must carry.
|
|
36
|
+
REMARK_EXCERPT_LENGTH = 40
|
|
37
|
+
|
|
38
|
+
# Express listings end with the schema-close token; a schema-doc
|
|
39
|
+
# page whose last token renders outside a listing element closed
|
|
40
|
+
# its <pre> early and leaked the tail as normal HTML.
|
|
41
|
+
END_SCHEMA_TOKEN = "END_SCHEMA;"
|
|
42
|
+
|
|
43
|
+
# Asciidoc constructs that must never render as page text
|
|
44
|
+
# (source-authoring residuals): inline macros plus the bare
|
|
45
|
+
# forced-line-break "+".
|
|
46
|
+
ASCIIDOC_MACROS_RE = /\b(?:image|link|include)::[^ \t\r\n]*\[\]/
|
|
47
|
+
LISTING_TAGS = %w[pre code].freeze
|
|
48
|
+
|
|
26
49
|
Report = Struct.new(:report, keyword_init: true) do
|
|
27
50
|
def broken_count
|
|
28
51
|
report["broken_links"].size + report["broken_fragments"].size
|
|
@@ -33,9 +56,15 @@ module Suma
|
|
|
33
56
|
end
|
|
34
57
|
end
|
|
35
58
|
|
|
36
|
-
def initialize(site_dir, check_fragments: true
|
|
59
|
+
def initialize(site_dir, check_fragments: true,
|
|
60
|
+
forbidden_path_prefixes: nil)
|
|
37
61
|
@site_dir = Pathname.new(site_dir)
|
|
38
62
|
@check_fragments = check_fragments
|
|
63
|
+
prefixes = Array(forbidden_path_prefixes)
|
|
64
|
+
@forbidden_path_prefixes =
|
|
65
|
+
prefixes.empty? ? DEFAULT_FORBIDDEN_PATH_PREFIXES : prefixes
|
|
66
|
+
@forbidden_path_re =
|
|
67
|
+
%r{#{Regexp.union(@forbidden_path_prefixes)}[\w./+-]*}
|
|
39
68
|
end
|
|
40
69
|
|
|
41
70
|
def verify
|
|
@@ -50,6 +79,13 @@ module Suma
|
|
|
50
79
|
doc = Leptris::XML.parse_html(content)
|
|
51
80
|
verify_intra_fragments(doc, rel_path, html_file, report)
|
|
52
81
|
verify_links(doc, rel_path, html_file, anchor_index, report)
|
|
82
|
+
check_build_dir_references(content, rel_path, report)
|
|
83
|
+
check_asciidoc_leakage(doc, rel_path, report)
|
|
84
|
+
check_page_inventory(doc, rel_path, report)
|
|
85
|
+
next unless schema_doc_page?(rel_path)
|
|
86
|
+
|
|
87
|
+
check_listing_completeness(doc, rel_path, report)
|
|
88
|
+
check_own_remark(doc, rel_path, report)
|
|
53
89
|
end
|
|
54
90
|
report["inventory"] = site_inventory
|
|
55
91
|
report["diagnostics"] = Diagnostics.count_site(@site_dir).to_h
|
|
@@ -71,6 +107,13 @@ module Suma
|
|
|
71
107
|
"resolved_intra_fragments" => 0,
|
|
72
108
|
"broken_intra_fragments" => [],
|
|
73
109
|
"missing_express_pages" => [],
|
|
110
|
+
"build_dir_pages" => [],
|
|
111
|
+
"build_dir_occurrences" => 0,
|
|
112
|
+
"incomplete_listings" => [],
|
|
113
|
+
"missing_schema_remarks" => [],
|
|
114
|
+
"asciidoc_leak_pages" => [],
|
|
115
|
+
"asciidoc_leak_occurrences" => 0,
|
|
116
|
+
"page_inventory" => {},
|
|
74
117
|
"inventory" => {},
|
|
75
118
|
"diagnostics" => {},
|
|
76
119
|
}
|
|
@@ -94,6 +137,107 @@ module Suma
|
|
|
94
137
|
}
|
|
95
138
|
end
|
|
96
139
|
|
|
140
|
+
# Build-directory leakage: every occurrence of a path under a
|
|
141
|
+
# forbidden prefix, page-level and occurrence-level.
|
|
142
|
+
def check_build_dir_references(content, rel_path, report)
|
|
143
|
+
hits = content.scan(@forbidden_path_re)
|
|
144
|
+
return if hits.empty?
|
|
145
|
+
|
|
146
|
+
report["build_dir_pages"] << rel_path
|
|
147
|
+
report["build_dir_occurrences"] += hits.size
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Asciidoc source constructs leaked as rendered page text,
|
|
151
|
+
# counted over text nodes only (markup-inside-attributes is not
|
|
152
|
+
# leakage).
|
|
153
|
+
def check_asciidoc_leakage(doc, rel_path, report)
|
|
154
|
+
leaks = doc.xpath("//text()").sum do |node|
|
|
155
|
+
text = node.text.to_s
|
|
156
|
+
text.scan(ASCIIDOC_MACROS_RE).size + (text.strip == "+" ? 1 : 0)
|
|
157
|
+
end
|
|
158
|
+
return if leaks.zero?
|
|
159
|
+
|
|
160
|
+
report["asciidoc_leak_pages"] << rel_path
|
|
161
|
+
report["asciidoc_leak_occurrences"] += leaks
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# The per-page inventory ratchet rides in the machine-readable
|
|
165
|
+
# report: image and link counts per page.
|
|
166
|
+
def check_page_inventory(doc, rel_path, report)
|
|
167
|
+
report["page_inventory"][rel_path] = {
|
|
168
|
+
"imgs" => doc.css("img").size,
|
|
169
|
+
"links" => doc.css("a[href]").size,
|
|
170
|
+
}
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Schema-doc pages only: a schema listing that closed early
|
|
174
|
+
# renders its tail as normal HTML, so the final END_SCHEMA; token
|
|
175
|
+
# no longer sits inside a listing element.
|
|
176
|
+
def check_listing_completeness(doc, rel_path, report)
|
|
177
|
+
return if listing_closes?(doc)
|
|
178
|
+
|
|
179
|
+
report["incomplete_listings"] << rel_path
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def listing_closes?(doc)
|
|
183
|
+
tokens = doc.xpath("//text()")
|
|
184
|
+
last = tokens.select do |node|
|
|
185
|
+
node.text&.include?(END_SCHEMA_TOKEN)
|
|
186
|
+
end.last
|
|
187
|
+
last && inside_listing?(last)
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def inside_listing?(node)
|
|
191
|
+
current = node.parent
|
|
192
|
+
while current
|
|
193
|
+
return true if LISTING_TAGS.include?(current.name)
|
|
194
|
+
|
|
195
|
+
current = current.parent
|
|
196
|
+
end
|
|
197
|
+
false
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# A schema whose parsed attachment carries its own remarks must
|
|
201
|
+
# publish them: the page text contains the whitespace-normalized
|
|
202
|
+
# head of the first remark. Schemas without an attachment on the
|
|
203
|
+
# site cannot be judged and are skipped.
|
|
204
|
+
def check_own_remark(doc, rel_path, report)
|
|
205
|
+
schema_id = File.basename(rel_path, ".html").delete_prefix("doc_")
|
|
206
|
+
remarks = schema_remarks[schema_id]
|
|
207
|
+
return if Array(remarks).empty?
|
|
208
|
+
|
|
209
|
+
excerpt = whitespace_norm(remarks.first)[0, REMARK_EXCERPT_LENGTH]
|
|
210
|
+
return if whitespace_norm(page_text(doc)).include?(excerpt)
|
|
211
|
+
|
|
212
|
+
report["missing_schema_remarks"] << schema_id
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# The page's whole text content (Leptris documents expose no
|
|
216
|
+
# #text of their own).
|
|
217
|
+
def page_text(doc)
|
|
218
|
+
doc.xpath("//text()").map { |node| node.text.to_s }.join(" ")
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def whitespace_norm(text)
|
|
222
|
+
text.to_s.gsub(/\s+/, " ").strip
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# One Expressir parse per schema attachment, reused across every
|
|
226
|
+
# schema-doc page of the site.
|
|
227
|
+
def schema_remarks
|
|
228
|
+
@schema_remarks ||= Dir.glob("#{@site_dir}/plain_schemas/**/*.exp")
|
|
229
|
+
.filter_map do |exp|
|
|
230
|
+
schema = Expressir::Express::Parser.from_file(exp).schemas.first
|
|
231
|
+
next unless schema
|
|
232
|
+
|
|
233
|
+
[File.basename(exp, ".exp"), schema.remarks]
|
|
234
|
+
end.to_h
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def schema_doc_page?(rel_path)
|
|
238
|
+
File.basename(rel_path).start_with?("doc_")
|
|
239
|
+
end
|
|
240
|
+
|
|
97
241
|
def html_files
|
|
98
242
|
Dir.glob("#{@site_dir}/**/*.html")
|
|
99
243
|
.reject { |f| f.end_with?(".err.html") }
|
data/lib/suma/delivery.rb
CHANGED
|
@@ -9,6 +9,7 @@ module Suma
|
|
|
9
9
|
autoload :Diagnostics, "suma/delivery/diagnostics"
|
|
10
10
|
autoload :Expectations, "suma/delivery/expectations"
|
|
11
11
|
autoload :Pack, "suma/delivery/pack"
|
|
12
|
+
autoload :PageInventory, "suma/delivery/page_inventory"
|
|
12
13
|
autoload :Plan, "suma/delivery/plan"
|
|
13
14
|
autoload :SiteVerifier, "suma/delivery/site_verifier"
|
|
14
15
|
autoload :Stitcher, "suma/delivery/stitcher"
|
data/lib/suma/schema_exporter.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "etc"
|
|
3
4
|
require "fileutils"
|
|
4
5
|
|
|
5
6
|
module Suma
|
|
@@ -42,7 +43,10 @@ module Suma
|
|
|
42
43
|
annotations: false,
|
|
43
44
|
create_zip: false,
|
|
44
45
|
structure: :preserve,
|
|
45
|
-
|
|
46
|
+
# Cold-parse parallelism by default: the parsanol parse is the
|
|
47
|
+
# dominant cost of a vanilla (no cache) build and is GVL-free
|
|
48
|
+
# (Rust), so N workers is a near-N-fold cut. Pass jobs: to pin it.
|
|
49
|
+
jobs: Etc.nprocessors,
|
|
46
50
|
}
|
|
47
51
|
end
|
|
48
52
|
|
|
@@ -20,6 +20,9 @@ module Suma
|
|
|
20
20
|
EXTENSIONS
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
# The listing method emits the complete source block — the
|
|
24
|
+
# [source%unnumbered,subs="+macros"] fence included — so the
|
|
25
|
+
# template carries no fence of its own.
|
|
23
26
|
def render(path_to_schema_yaml)
|
|
24
27
|
<<~ADOC
|
|
25
28
|
= #{schema_id}
|
|
@@ -36,10 +39,7 @@ module Suma
|
|
|
36
39
|
[%unnumbered,type=express]
|
|
37
40
|
== #{schema_id} #{rendered_anchors}
|
|
38
41
|
|
|
39
|
-
|
|
40
|
-
--
|
|
41
|
-
{{ schema.formatted_hyperlinked_adoc }}
|
|
42
|
-
--
|
|
42
|
+
{{ schema.formatted_hyperlinked_adoc_listing }}
|
|
43
43
|
{% endfor %}
|
|
44
44
|
----
|
|
45
45
|
|
|
@@ -20,6 +20,9 @@ module Suma
|
|
|
20
20
|
EXTENSIONS
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
# The listing method emits the complete source block — the
|
|
24
|
+
# [source%unnumbered,subs="+macros"] fence included — so the
|
|
25
|
+
# template carries no fence of its own.
|
|
23
26
|
def render(path_to_schema_yaml)
|
|
24
27
|
<<~ADOC
|
|
25
28
|
= #{schema_id}
|
|
@@ -36,10 +39,7 @@ module Suma
|
|
|
36
39
|
[%unnumbered,type=express]
|
|
37
40
|
== #{schema_id} #{rendered_anchors}
|
|
38
41
|
|
|
39
|
-
|
|
40
|
-
--
|
|
41
|
-
{{ schema.formatted_hyperlinked_adoc }}
|
|
42
|
-
--
|
|
42
|
+
{{ schema.formatted_hyperlinked_adoc_listing }}
|
|
43
43
|
{% endfor %}
|
|
44
44
|
----
|
|
45
45
|
|
data/lib/suma/term_extractor.rb
CHANGED
|
@@ -6,6 +6,10 @@ require "glossarist"
|
|
|
6
6
|
|
|
7
7
|
module Suma
|
|
8
8
|
class TermExtractor
|
|
9
|
+
# Emitted in place of a supertype cross-reference the interface walk
|
|
10
|
+
# cannot resolve: a URN would fabricate an owner for the declaration.
|
|
11
|
+
UNRESOLVED_TARGET = "target unresolved"
|
|
12
|
+
|
|
9
13
|
def initialize(schema_manifest_file, output_path, urn:,
|
|
10
14
|
language_code: "eng", profile: nil, edition: nil,
|
|
11
15
|
language_codes: nil)
|
|
@@ -288,8 +292,7 @@ module Suma
|
|
|
288
292
|
"#{entity_name_to_text(entity.id)} #{entity_ref}"
|
|
289
293
|
else
|
|
290
294
|
entity_subtypes = subtypes.map do |e|
|
|
291
|
-
|
|
292
|
-
urn_mention(express_entity_urn("#{owner}.#{e.id}"), e.id)
|
|
295
|
+
supertype_mention(schema, e.id)
|
|
293
296
|
end
|
|
294
297
|
|
|
295
298
|
"#{entity_type} that is a type of " \
|
|
@@ -299,21 +302,30 @@ module Suma
|
|
|
299
302
|
end
|
|
300
303
|
end
|
|
301
304
|
|
|
302
|
-
# The
|
|
303
|
-
#
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
305
|
+
# The citation for a supertype named in +schema+: qualified with the
|
|
306
|
+
# declaring schema resolved through the interface closure, or the
|
|
307
|
+
# literal "target unresolved". A name the closure does not resolve —
|
|
308
|
+
# absent, ambiguous, or case-variant (the walk is exact-match, and
|
|
309
|
+
# EXPRESS spellings vary) — has no verifiable declarer; the manifest
|
|
310
|
+
# scan that used to guess one emitted citations that mis-cited the
|
|
311
|
+
# declaration, so no xref is emitted at all (WG 12 position on the
|
|
312
|
+
# 0.5.42 audit, iso-10303#835).
|
|
313
|
+
def supertype_mention(schema, entity_id)
|
|
314
|
+
owner = declaring_schema_id(schema, entity_id)
|
|
315
|
+
return UNRESOLVED_TARGET unless owner
|
|
316
|
+
|
|
317
|
+
urn_mention(express_entity_urn("#{owner}.#{entity_id}"), entity_id)
|
|
307
318
|
end
|
|
308
319
|
|
|
309
320
|
# The schema an imported declaration is declared in, resolved through
|
|
310
321
|
# +schema+'s USE-FROM closure — the same resolution the source remark
|
|
311
322
|
# xrefs carry: a local declaration wins; otherwise the USE FROM edges
|
|
312
323
|
# (item-naming ones first, then bare ones) are walked transitively,
|
|
313
|
-
# and a single distinct declaring schema wins. nil
|
|
314
|
-
#
|
|
315
|
-
#
|
|
316
|
-
#
|
|
324
|
+
# and a single distinct declaring schema wins. nil means unresolved:
|
|
325
|
+
# supertype_mention then emits the literal "target unresolved" rather
|
|
326
|
+
# than qualifying the citation with any schema (iso-10303#835: the
|
|
327
|
+
# declarer can sit more than one USE FROM away, and sibling branches
|
|
328
|
+
# must all be searched before settling).
|
|
317
329
|
def declaring_schema_id(schema, entity_id)
|
|
318
330
|
# The closure walk is deterministic per (schema, entity); the
|
|
319
331
|
# corpus repeats the same lookups across thousands of concepts and
|
|
@@ -328,39 +340,6 @@ module Suma
|
|
|
328
340
|
end
|
|
329
341
|
end
|
|
330
342
|
|
|
331
|
-
# When the closure walk finds no declarer, the declaration may sit
|
|
332
|
-
# in a sibling schema with no interface edge to the citing one: the
|
|
333
|
-
# _arm/_mim pairs cite across without USE FROM/REFERENCE FROM, so
|
|
334
|
-
# `X_arm.Y` lands on a concept declared under `X_mim` (TRThurman's
|
|
335
|
-
# 148 distinct wrong-qualifier targets, iso-10303#835 /
|
|
336
|
-
# metanorma/suma#197). Scan the manifest's schemas for the entity
|
|
337
|
-
# name; exactly one declarer resolves, several stay nil.
|
|
338
|
-
def sibling_declaring_schema_id(entity_id)
|
|
339
|
-
@sibling_cache ||= {}
|
|
340
|
-
@sibling_cache.fetch(entity_id) do
|
|
341
|
-
declarers = declarers_in_manifest(entity_id)
|
|
342
|
-
unique_declarer(declarers)
|
|
343
|
-
end
|
|
344
|
-
end
|
|
345
|
-
|
|
346
|
-
# The manifest schemas whose entity collection declares +entity_id+.
|
|
347
|
-
def declarers_in_manifest(entity_id)
|
|
348
|
-
schema_manifest_paths.keys.filter_map do |id|
|
|
349
|
-
loaded = loaded_schema(id)
|
|
350
|
-
declares_entity?(loaded, entity_id) ? id : nil
|
|
351
|
-
end
|
|
352
|
-
end
|
|
353
|
-
|
|
354
|
-
def declares_entity?(schema, entity_id)
|
|
355
|
-
schema&.entities&.any? { |e| e.id == entity_id } ? true : false
|
|
356
|
-
end
|
|
357
|
-
|
|
358
|
-
# The single distinct declarer, or nil when none or several — an
|
|
359
|
-
# ambiguous name never resolves to a guess.
|
|
360
|
-
def unique_declarer(declarers)
|
|
361
|
-
declarers.uniq.one? ? declarers.first : nil
|
|
362
|
-
end
|
|
363
|
-
|
|
364
343
|
# Distinct schema ids within +schema+'s USE-FROM closure declaring
|
|
365
344
|
# +entity_id+. An interface whose item list names the declaration
|
|
366
345
|
# claims it outright (that is what the item list means in EXPRESS) —
|
data/lib/suma/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: suma
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.43
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-11-
|
|
11
|
+
date: 2026-11-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: base64
|
|
@@ -247,6 +247,7 @@ files:
|
|
|
247
247
|
- lib/suma/delivery/expectations.rb
|
|
248
248
|
- lib/suma/delivery/frame_nav.rb
|
|
249
249
|
- lib/suma/delivery/pack.rb
|
|
250
|
+
- lib/suma/delivery/page_inventory.rb
|
|
250
251
|
- lib/suma/delivery/plan.rb
|
|
251
252
|
- lib/suma/delivery/site_verifier.rb
|
|
252
253
|
- lib/suma/delivery/stitcher.rb
|