suma 0.5.33 → 0.5.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +2 -0
- data/lib/suma/cli/delivery.rb +55 -7
- data/lib/suma/delivery/cross_page_fragments.rb +134 -0
- data/lib/suma/delivery/expectations.rb +72 -0
- data/lib/suma/delivery/site_verifier.rb +34 -3
- data/lib/suma/delivery.rb +2 -0
- data/lib/suma/version.rb +1 -1
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2555998b3b588d4c0d92ff306048f844051686cea1b36e17e40c6be9df389e25
|
|
4
|
+
data.tar.gz: e83c32dbeddc593bf5b640de88be5cbd1463a305f01bb28ad7ea4ca66e24fe30
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e1091d2b9af8da12d387d0fcb8dc26db93e7d6f87481e065cc28a8c7b8b97800943b32330ff9e6ae9b2166a3eab3cafbb3d7648abd2902de71b6a2931fefdb59
|
|
7
|
+
data.tar.gz: 57b64f147c71de0f474be81406624611d46232440f62505cee9f2d316edcd0286230d00e806b8d57a633526823718d226972c2df14b03f82f414f666099f4ffe
|
data/.rubocop_todo.yml
CHANGED
|
@@ -276,6 +276,7 @@ RSpec/ContextWording:
|
|
|
276
276
|
# Configuration parameters: Max, CountAsOne.
|
|
277
277
|
RSpec/ExampleLength:
|
|
278
278
|
Exclude:
|
|
279
|
+
- 'spec/suma/delivery/expectations_spec.rb'
|
|
279
280
|
- 'spec/suma/anchor_validator_spec.rb'
|
|
280
281
|
- 'spec/suma/artifact_server_spec.rb'
|
|
281
282
|
- 'spec/suma/build_plan_spec.rb'
|
|
@@ -375,6 +376,7 @@ RSpec/MessageSpies:
|
|
|
375
376
|
# Configuration parameters: Max.
|
|
376
377
|
RSpec/MultipleExpectations:
|
|
377
378
|
Exclude:
|
|
379
|
+
- 'spec/suma/delivery/expectations_spec.rb'
|
|
378
380
|
- 'spec/suma/artifact_server_spec.rb'
|
|
379
381
|
- 'spec/suma/change_classification_spec.rb'
|
|
380
382
|
- 'spec/suma/change_impact_spec.rb'
|
data/lib/suma/cli/delivery.rb
CHANGED
|
@@ -21,13 +21,35 @@ module Suma
|
|
|
21
21
|
desc: "Also write the report as JSON to this path"
|
|
22
22
|
option :no_fragments, type: :boolean, default: false,
|
|
23
23
|
desc: "Skip fragment checking (file existence only)"
|
|
24
|
+
option :expectations, type: :string,
|
|
25
|
+
desc: "YAML manifest of site-shape thresholds " \
|
|
26
|
+
"(sections_min, index_pages, " \
|
|
27
|
+
"schema_attachments_min, schema_docs_min, " \
|
|
28
|
+
"missing_express_max, broken_links_max); " \
|
|
29
|
+
"violations fail the verify"
|
|
24
30
|
|
|
25
31
|
def verify(site_dir)
|
|
26
32
|
result = Suma::Delivery::SiteVerifier.new(site_dir,
|
|
27
33
|
**verify_options).verify
|
|
28
34
|
log_verify(result)
|
|
35
|
+
failures = expectation_failures(result)
|
|
29
36
|
raise "broken links found (#{result.broken_count})" unless
|
|
30
37
|
result.broken_count.zero?
|
|
38
|
+
raise "expectations violated: #{failures.join('; ')}" unless
|
|
39
|
+
failures.empty?
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
desc "resolve-fragments SITE_DIR",
|
|
43
|
+
"Assembly-time cross-page fragment resolution: repoint " \
|
|
44
|
+
"same-page fragment hrefs whose target renders on exactly " \
|
|
45
|
+
"one other page of the stitched site. Ambiguous matches are " \
|
|
46
|
+
"left untouched."
|
|
47
|
+
def resolve_fragments(site_dir)
|
|
48
|
+
result = Suma::Delivery::CrossPageFragments.new(site_dir).repair
|
|
49
|
+
Utils.log "Fragments: #{result[:own]} own-page fine, " \
|
|
50
|
+
"#{result[:remapped]} remapped cross-page, " \
|
|
51
|
+
"#{result[:ambiguous]} ambiguous left, " \
|
|
52
|
+
"#{result[:unresolved]} unresolved"
|
|
31
53
|
end
|
|
32
54
|
|
|
33
55
|
desc "stitch BUNDLES_DIR",
|
|
@@ -112,14 +134,40 @@ module Suma
|
|
|
112
134
|
{ check_fragments: !options[:no_fragments] }
|
|
113
135
|
end
|
|
114
136
|
|
|
137
|
+
def load_expectations
|
|
138
|
+
return unless options[:expectations]
|
|
139
|
+
|
|
140
|
+
Suma::Delivery::Expectations
|
|
141
|
+
.from_yaml(File.read(options[:expectations]))
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def expectation_failures(result)
|
|
145
|
+
expectations = load_expectations or return []
|
|
146
|
+
expectations.violations(result.report["inventory"],
|
|
147
|
+
result.report["broken_links"])
|
|
148
|
+
end
|
|
149
|
+
|
|
115
150
|
def log_verify(result)
|
|
116
|
-
report
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
151
|
+
write_report(result.report) if options[:out]
|
|
152
|
+
Utils.log verify_summary(result)
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def verify_summary(result)
|
|
156
|
+
r = result.report
|
|
157
|
+
"Links: #{r['total_links']} total, #{r['broken_links'].size} " \
|
|
158
|
+
"broken; fragments: #{r['total_fragments']} total, " \
|
|
159
|
+
"#{r['broken_fragments'].size} broken; " \
|
|
160
|
+
"#{r['cross_document_links']} cross-document; " \
|
|
161
|
+
"#{inventory_summary(r)}"
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def inventory_summary(report)
|
|
165
|
+
inv = report["inventory"]
|
|
166
|
+
"site: sections=#{inv['sections']} " \
|
|
167
|
+
"indexes=#{inv['index_pages']} " \
|
|
168
|
+
"attachments=#{inv['schema_attachments']} " \
|
|
169
|
+
"schema_docs=#{inv['schema_docs']}; " \
|
|
170
|
+
"missing-express pages=#{report['missing_express_pages'].size}"
|
|
123
171
|
end
|
|
124
172
|
|
|
125
173
|
def write_report(report)
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pathname"
|
|
4
|
+
require "leptris"
|
|
5
|
+
|
|
6
|
+
module Suma
|
|
7
|
+
module Delivery
|
|
8
|
+
# Assembly-time cross-page fragment resolution (the delivery
|
|
9
|
+
# correctness model's final pass): per-document rendering writes
|
|
10
|
+
# term-fragment and svgmap-hotspot hrefs page-locally, and the
|
|
11
|
+
# unioned site holds their targets on other pages. Two resolution
|
|
12
|
+
# rules, both requiring a unique site-wide match:
|
|
13
|
+
#
|
|
14
|
+
# 1. the fragment id exists on exactly one other page — the href
|
|
15
|
+
# repoints at that page;
|
|
16
|
+
# 2. the fragment names a declaration whose rendered anchor carries
|
|
17
|
+
# the stamped form `<scope>_ISO_…` with scope ending in
|
|
18
|
+
# `.<fragment>` (or equal to it) — the href repoints at the page
|
|
19
|
+
# rendering that anchor, at the full stamped id.
|
|
20
|
+
#
|
|
21
|
+
# Ambiguous matches (two or more candidate pages/anchors) stay
|
|
22
|
+
# untouched: guessing a target would be wrong exactly like an
|
|
23
|
+
# unresolved one.
|
|
24
|
+
class CrossPageFragments
|
|
25
|
+
SCOPE_PART = /[A-Za-z0-9-]+_[0-9]+/
|
|
26
|
+
# `<scope>_ISO_<doc>_<year>` — the scope captures as `schema` or
|
|
27
|
+
# `schema.entity`.
|
|
28
|
+
SCOPED_ID_RE = /\A([a-z][a-z0-9_]*(?:\.[a-z0-9_.]+)?)_ISO_#{SCOPE_PART}/
|
|
29
|
+
|
|
30
|
+
Result = Struct.new(:own, :remapped, :ambiguous, :unresolved,
|
|
31
|
+
keyword_init: true)
|
|
32
|
+
|
|
33
|
+
def initialize(site_dir)
|
|
34
|
+
@site_dir = Pathname.new(site_dir)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def repair
|
|
38
|
+
pages = html_pages
|
|
39
|
+
id_index = index_ids(pages)
|
|
40
|
+
counts = Hash.new(0)
|
|
41
|
+
pages.each do |path, doc|
|
|
42
|
+
remap_page(doc, path, id_index, counts)
|
|
43
|
+
path.write(doc.to_html)
|
|
44
|
+
end
|
|
45
|
+
Result.new(**counts.slice(:own, :remapped, :ambiguous, :unresolved))
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def remap_page(doc, path, id_index, counts)
|
|
51
|
+
doc.css("a[href^='#']").each do |a|
|
|
52
|
+
status, href = resolve(a["href"].to_s.delete_prefix("#"),
|
|
53
|
+
path, id_index)
|
|
54
|
+
status = :unresolved if status.nil?
|
|
55
|
+
counts[status] += 1
|
|
56
|
+
a["href"] = href if href
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# [status, href-or-nil]: :own = the target id is on the citing
|
|
61
|
+
# page (fine as written); :ambiguous = several candidates;
|
|
62
|
+
# :unresolved = no candidate; :remapped carries the new href.
|
|
63
|
+
def resolve(frag, path, id_index)
|
|
64
|
+
exact = id_index[frag]
|
|
65
|
+
if exact
|
|
66
|
+
uniq = exact[:pages].uniq
|
|
67
|
+
return [:own] if uniq == [path]
|
|
68
|
+
return [:ambiguous] if uniq.size > 1
|
|
69
|
+
|
|
70
|
+
return [:remapped, cross_href(path, uniq.first, frag)]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
scoped_resolve(frag, path, id_index)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def scoped_resolve(frag, path, id_index)
|
|
77
|
+
candidates = matching_scopes(frag, path, id_index)
|
|
78
|
+
return [:ambiguous] if candidates.size > 1
|
|
79
|
+
|
|
80
|
+
id, pages = candidates.first
|
|
81
|
+
return [:unresolved, nil] unless id
|
|
82
|
+
|
|
83
|
+
[:remapped, cross_href(path, pages.first, id)]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# [id, pages] pairs for stamped anchors whose scope names the
|
|
87
|
+
# fragment, rendered on some page other than the citing one.
|
|
88
|
+
def matching_scopes(frag, path, id_index)
|
|
89
|
+
id_index.filter_map do |id, entry|
|
|
90
|
+
pages = entry[:pages].uniq
|
|
91
|
+
next if pages == [path]
|
|
92
|
+
|
|
93
|
+
scope = scope_of(id)
|
|
94
|
+
next unless scope == frag || scope.end_with?(".#{frag}")
|
|
95
|
+
|
|
96
|
+
[id, pages]
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def scope_of(id)
|
|
101
|
+
SCOPED_ID_RE.match(id)&.[](1)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def cross_href(path, page, fragment)
|
|
105
|
+
"#{relative_href(path, page)}##{fragment}"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def relative_href(from, to)
|
|
109
|
+
from.dirname.join(to.basename).relative_path_from(from.dirname).to_s
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def html_pages
|
|
113
|
+
@html_pages ||= @site_dir.glob("**/*.html")
|
|
114
|
+
.reject { |p| p.basename.to_s.end_with?(".err.html") }
|
|
115
|
+
.to_h { |p| [p, Leptris::XML.parse_html(p.read)] }
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# fragment id → { pages: [path, …] }. A plain hash: reads of
|
|
119
|
+
# unknown ids must not insert (matching_scopes iterates the index
|
|
120
|
+
# afterwards).
|
|
121
|
+
def index_ids(pages)
|
|
122
|
+
index = {}
|
|
123
|
+
pages.each do |path, doc|
|
|
124
|
+
doc.css("[id]").each do |el|
|
|
125
|
+
next if (id = el["id"]).nil? || id.empty?
|
|
126
|
+
|
|
127
|
+
(index[id] ||= { pages: [] })[:pages] << path
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
index
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Suma
|
|
4
|
+
module Delivery
|
|
5
|
+
# Corpus-side gate thresholds for `delivery verify --expectations`:
|
|
6
|
+
# the measured site-shape conditions as data, so the calling workflow
|
|
7
|
+
# states its delivery contract in one manifest instead of hand-rolled
|
|
8
|
+
# shell checks. Unset attributes are not checked.
|
|
9
|
+
class Expectations < Lutaml::Model::Serializable
|
|
10
|
+
attribute :sections_min, :integer
|
|
11
|
+
attribute :index_pages, :integer
|
|
12
|
+
attribute :schema_attachments_min, :integer
|
|
13
|
+
attribute :schema_docs_min, :integer
|
|
14
|
+
attribute :missing_express_max, :integer, default: -> { 0 }
|
|
15
|
+
attribute :broken_links_max, :integer, default: -> { 0 }
|
|
16
|
+
|
|
17
|
+
key_value do
|
|
18
|
+
map "sections_min", to: :sections_min
|
|
19
|
+
map "index_pages", to: :index_pages
|
|
20
|
+
map "schema_attachments_min", to: :schema_attachments_min
|
|
21
|
+
map "schema_docs_min", to: :schema_docs_min
|
|
22
|
+
map "missing_express_max", to: :missing_express_max
|
|
23
|
+
map "broken_links_max", to: :broken_links_max
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Human-readable violation lines for every threshold the +inventory+
|
|
27
|
+
# and +broken_links+ breach; unset thresholds stay silent. The name
|
|
28
|
+
# suffix is the comparison: `_min` floors, `_max` caps, a bare name
|
|
29
|
+
# (index_pages) is exact.
|
|
30
|
+
def violations(inventory, broken_links)
|
|
31
|
+
actuals(inventory, broken_links).filter_map do |name, actual|
|
|
32
|
+
limit = public_send(name) or next
|
|
33
|
+
relation = relation_of(name)
|
|
34
|
+
next if relation_met?(relation, actual, limit)
|
|
35
|
+
|
|
36
|
+
format("%<name>s: got %<actual>d, expected %<relation>s " \
|
|
37
|
+
"%<limit>d", name: name, actual: actual,
|
|
38
|
+
relation: relation, limit: limit)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def relation_of(name)
|
|
45
|
+
return "at least" if name.end_with?("_min")
|
|
46
|
+
return "at most" if name.end_with?("_max")
|
|
47
|
+
|
|
48
|
+
"exactly"
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def relation_met?(relation, actual, limit)
|
|
52
|
+
case relation
|
|
53
|
+
when "at least" then actual >= limit
|
|
54
|
+
when "at most" then actual <= limit
|
|
55
|
+
else actual == limit
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Every threshold's observed value, keyed by threshold name.
|
|
60
|
+
def actuals(inventory, broken_links)
|
|
61
|
+
{
|
|
62
|
+
"sections_min" => inventory["sections"],
|
|
63
|
+
"index_pages" => inventory["index_pages"],
|
|
64
|
+
"schema_attachments_min" => inventory["schema_attachments"],
|
|
65
|
+
"schema_docs_min" => inventory["schema_docs"],
|
|
66
|
+
"missing_express_max" => inventory["missing_express_pages"].size,
|
|
67
|
+
"broken_links_max" => broken_links.size,
|
|
68
|
+
}
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -18,7 +18,10 @@ module Suma
|
|
|
18
18
|
# without making it clearer.
|
|
19
19
|
# rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
20
20
|
SCHEME_RE = %r{\A(?:https?:|mailto:|javascript:|data:)}
|
|
21
|
-
|
|
21
|
+
|
|
22
|
+
# Metanorma's unresolved-citation rendering prefix; a page carrying
|
|
23
|
+
# it cites an express reference the corpus never resolved.
|
|
24
|
+
MISSING_EXPRESS_MARKER = "Missing:express"
|
|
22
25
|
|
|
23
26
|
Report = Struct.new(:report, keyword_init: true) do
|
|
24
27
|
def broken_count
|
|
@@ -40,10 +43,15 @@ module Suma
|
|
|
40
43
|
anchor_index = build_anchor_index
|
|
41
44
|
html_files.each do |html_file|
|
|
42
45
|
rel_path = html_file.relative_path_from(@site_dir).to_s
|
|
43
|
-
|
|
46
|
+
content = html_file.read
|
|
47
|
+
if content.include?(MISSING_EXPRESS_MARKER)
|
|
48
|
+
report["missing_express_pages"] << rel_path
|
|
49
|
+
end
|
|
50
|
+
doc = Leptris::XML.parse_html(content)
|
|
44
51
|
verify_intra_fragments(doc, rel_path, html_file, report)
|
|
45
52
|
verify_links(doc, rel_path, html_file, anchor_index, report)
|
|
46
53
|
end
|
|
54
|
+
report["inventory"] = site_inventory
|
|
47
55
|
Report.new(report: report)
|
|
48
56
|
end
|
|
49
57
|
|
|
@@ -61,6 +69,26 @@ module Suma
|
|
|
61
69
|
"total_intra_fragments" => 0,
|
|
62
70
|
"resolved_intra_fragments" => 0,
|
|
63
71
|
"broken_intra_fragments" => [],
|
|
72
|
+
"missing_express_pages" => [],
|
|
73
|
+
"inventory" => {},
|
|
74
|
+
}
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# The stitched site's shape: the counts the delivery contract is
|
|
78
|
+
# stated against (section pages, index pages, schema attachments,
|
|
79
|
+
# schema doc pages).
|
|
80
|
+
def site_inventory
|
|
81
|
+
{
|
|
82
|
+
"sections" => Dir.glob("#{@site_dir}/**/member_*.xml.*.html")
|
|
83
|
+
.count { |f| !f.end_with?(".err.html") },
|
|
84
|
+
"index_pages" => Dir.glob("#{@site_dir}/**/*_index.html").size,
|
|
85
|
+
"schema_attachments" =>
|
|
86
|
+
Dir.glob("#{@site_dir}/plain_schemas/**/*.exp").size,
|
|
87
|
+
"schema_docs" => if Dir.exist?("#{@site_dir}/schema_docs")
|
|
88
|
+
Dir.children("#{@site_dir}/schema_docs").size
|
|
89
|
+
else
|
|
90
|
+
0
|
|
91
|
+
end,
|
|
64
92
|
}
|
|
65
93
|
end
|
|
66
94
|
|
|
@@ -104,7 +132,10 @@ module Suma
|
|
|
104
132
|
href = a["href"].to_s.strip
|
|
105
133
|
next if href.empty? || href.start_with?("#")
|
|
106
134
|
next if href.match?(SCHEME_RE)
|
|
107
|
-
|
|
135
|
+
|
|
136
|
+
# Bare integers are counted, not skipped: an unbound svgmap
|
|
137
|
+
# hotspot emits its ordinal as the href, and a link to a
|
|
138
|
+
# nonexistent file "12" is dead whatever the key form.
|
|
108
139
|
|
|
109
140
|
file_part, fragment = href.split("#", 2)
|
|
110
141
|
file_part = file_part.delete_prefix("./")
|
data/lib/suma/delivery.rb
CHANGED
|
@@ -5,6 +5,8 @@ module Suma
|
|
|
5
5
|
# per-part sites and resolve their unresolved cross-document references.
|
|
6
6
|
module Delivery
|
|
7
7
|
autoload :FrameNav, "suma/delivery/frame_nav"
|
|
8
|
+
autoload :CrossPageFragments, "suma/delivery/cross_page_fragments"
|
|
9
|
+
autoload :Expectations, "suma/delivery/expectations"
|
|
8
10
|
autoload :Pack, "suma/delivery/pack"
|
|
9
11
|
autoload :Plan, "suma/delivery/plan"
|
|
10
12
|
autoload :SiteVerifier, "suma/delivery/site_verifier"
|
data/lib/suma/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: suma
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.35
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
@@ -242,6 +242,8 @@ files:
|
|
|
242
242
|
- lib/suma/collection_manifest.rb
|
|
243
243
|
- lib/suma/corpus_graph.rb
|
|
244
244
|
- lib/suma/delivery.rb
|
|
245
|
+
- lib/suma/delivery/cross_page_fragments.rb
|
|
246
|
+
- lib/suma/delivery/expectations.rb
|
|
245
247
|
- lib/suma/delivery/frame_nav.rb
|
|
246
248
|
- lib/suma/delivery/pack.rb
|
|
247
249
|
- lib/suma/delivery/plan.rb
|