suma 0.5.33 → 0.5.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: bd2e4a074e8d3b0ab67fd577a72084f05b6727c472b27d8372b8f22d0c98851c
4
- data.tar.gz: fd67ff08c1e9607ec9e24ed66100dda235bfea0805b42427c83199313a868632
3
+ metadata.gz: 2555998b3b588d4c0d92ff306048f844051686cea1b36e17e40c6be9df389e25
4
+ data.tar.gz: e83c32dbeddc593bf5b640de88be5cbd1463a305f01bb28ad7ea4ca66e24fe30
5
5
  SHA512:
6
- metadata.gz: 3017ab40298248f60f38a32b25842535a47d72c36603de5c6c49b55d1a269341c497752e06c5f2bb372d0830155a49ecda7429310f0ffc44865ea8792398ab7b
7
- data.tar.gz: 011e091b656afb963cd9c12156588a1fbf90a40ca08071c91ce58b20df3b8ba71464ccc075ca9d2486d263ac20e2400d5ac8b942d97e9d043566d2b3feee85f5
6
+ metadata.gz: e1091d2b9af8da12d387d0fcb8dc26db93e7d6f87481e065cc28a8c7b8b97800943b32330ff9e6ae9b2166a3eab3cafbb3d7648abd2902de71b6a2931fefdb59
7
+ data.tar.gz: 57b64f147c71de0f474be81406624611d46232440f62505cee9f2d316edcd0286230d00e806b8d57a633526823718d226972c2df14b03f82f414f666099f4ffe
data/.rubocop_todo.yml CHANGED
@@ -276,6 +276,7 @@ RSpec/ContextWording:
276
276
  # Configuration parameters: Max, CountAsOne.
277
277
  RSpec/ExampleLength:
278
278
  Exclude:
279
+ - 'spec/suma/delivery/expectations_spec.rb'
279
280
  - 'spec/suma/anchor_validator_spec.rb'
280
281
  - 'spec/suma/artifact_server_spec.rb'
281
282
  - 'spec/suma/build_plan_spec.rb'
@@ -375,6 +376,7 @@ RSpec/MessageSpies:
375
376
  # Configuration parameters: Max.
376
377
  RSpec/MultipleExpectations:
377
378
  Exclude:
379
+ - 'spec/suma/delivery/expectations_spec.rb'
378
380
  - 'spec/suma/artifact_server_spec.rb'
379
381
  - 'spec/suma/change_classification_spec.rb'
380
382
  - 'spec/suma/change_impact_spec.rb'
@@ -21,13 +21,35 @@ module Suma
21
21
  desc: "Also write the report as JSON to this path"
22
22
  option :no_fragments, type: :boolean, default: false,
23
23
  desc: "Skip fragment checking (file existence only)"
24
+ option :expectations, type: :string,
25
+ desc: "YAML manifest of site-shape thresholds " \
26
+ "(sections_min, index_pages, " \
27
+ "schema_attachments_min, schema_docs_min, " \
28
+ "missing_express_max, broken_links_max); " \
29
+ "violations fail the verify"
24
30
 
25
31
  def verify(site_dir)
26
32
  result = Suma::Delivery::SiteVerifier.new(site_dir,
27
33
  **verify_options).verify
28
34
  log_verify(result)
35
+ failures = expectation_failures(result)
29
36
  raise "broken links found (#{result.broken_count})" unless
30
37
  result.broken_count.zero?
38
+ raise "expectations violated: #{failures.join('; ')}" unless
39
+ failures.empty?
40
+ end
41
+
42
+ desc "resolve-fragments SITE_DIR",
43
+ "Assembly-time cross-page fragment resolution: repoint " \
44
+ "same-page fragment hrefs whose target renders on exactly " \
45
+ "one other page of the stitched site. Ambiguous matches are " \
46
+ "left untouched."
47
+ def resolve_fragments(site_dir)
48
+ result = Suma::Delivery::CrossPageFragments.new(site_dir).repair
49
+ Utils.log "Fragments: #{result[:own]} own-page fine, " \
50
+ "#{result[:remapped]} remapped cross-page, " \
51
+ "#{result[:ambiguous]} ambiguous left, " \
52
+ "#{result[:unresolved]} unresolved"
31
53
  end
32
54
 
33
55
  desc "stitch BUNDLES_DIR",
@@ -112,14 +134,40 @@ module Suma
112
134
  { check_fragments: !options[:no_fragments] }
113
135
  end
114
136
 
137
+ def load_expectations
138
+ return unless options[:expectations]
139
+
140
+ Suma::Delivery::Expectations
141
+ .from_yaml(File.read(options[:expectations]))
142
+ end
143
+
144
+ def expectation_failures(result)
145
+ expectations = load_expectations or return []
146
+ expectations.violations(result.report["inventory"],
147
+ result.report["broken_links"])
148
+ end
149
+
115
150
  def log_verify(result)
116
- report = result.report
117
- write_report(report) if options[:out]
118
- Utils.log "Links: #{report['total_links']} total, " \
119
- "#{report['broken_links'].size} broken; " \
120
- "fragments: #{report['total_fragments']} total, " \
121
- "#{report['broken_fragments'].size} broken; " \
122
- "#{report['cross_document_links']} cross-document"
151
+ write_report(result.report) if options[:out]
152
+ Utils.log verify_summary(result)
153
+ end
154
+
155
+ def verify_summary(result)
156
+ r = result.report
157
+ "Links: #{r['total_links']} total, #{r['broken_links'].size} " \
158
+ "broken; fragments: #{r['total_fragments']} total, " \
159
+ "#{r['broken_fragments'].size} broken; " \
160
+ "#{r['cross_document_links']} cross-document; " \
161
+ "#{inventory_summary(r)}"
162
+ end
163
+
164
+ def inventory_summary(report)
165
+ inv = report["inventory"]
166
+ "site: sections=#{inv['sections']} " \
167
+ "indexes=#{inv['index_pages']} " \
168
+ "attachments=#{inv['schema_attachments']} " \
169
+ "schema_docs=#{inv['schema_docs']}; " \
170
+ "missing-express pages=#{report['missing_express_pages'].size}"
123
171
  end
124
172
 
125
173
  def write_report(report)
@@ -0,0 +1,134 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "pathname"
4
+ require "leptris"
5
+
6
+ module Suma
7
+ module Delivery
8
+ # Assembly-time cross-page fragment resolution (the delivery
9
+ # correctness model's final pass): per-document rendering writes
10
+ # term-fragment and svgmap-hotspot hrefs page-locally, and the
11
+ # unioned site holds their targets on other pages. Two resolution
12
+ # rules, both requiring a unique site-wide match:
13
+ #
14
+ # 1. the fragment id exists on exactly one other page — the href
15
+ # repoints at that page;
16
+ # 2. the fragment names a declaration whose rendered anchor carries
17
+ # the stamped form `<scope>_ISO_…` with scope ending in
18
+ # `.<fragment>` (or equal to it) — the href repoints at the page
19
+ # rendering that anchor, at the full stamped id.
20
+ #
21
+ # Ambiguous matches (two or more candidate pages/anchors) stay
22
+ # untouched: guessing a target would be wrong exactly like an
23
+ # unresolved one.
24
+ class CrossPageFragments
25
+ SCOPE_PART = /[A-Za-z0-9-]+_[0-9]+/
26
+ # `<scope>_ISO_<doc>_<year>` — the scope captures as `schema` or
27
+ # `schema.entity`.
28
+ SCOPED_ID_RE = /\A([a-z][a-z0-9_]*(?:\.[a-z0-9_.]+)?)_ISO_#{SCOPE_PART}/
29
+
30
+ Result = Struct.new(:own, :remapped, :ambiguous, :unresolved,
31
+ keyword_init: true)
32
+
33
+ def initialize(site_dir)
34
+ @site_dir = Pathname.new(site_dir)
35
+ end
36
+
37
+ def repair
38
+ pages = html_pages
39
+ id_index = index_ids(pages)
40
+ counts = Hash.new(0)
41
+ pages.each do |path, doc|
42
+ remap_page(doc, path, id_index, counts)
43
+ path.write(doc.to_html)
44
+ end
45
+ Result.new(**counts.slice(:own, :remapped, :ambiguous, :unresolved))
46
+ end
47
+
48
+ private
49
+
50
+ def remap_page(doc, path, id_index, counts)
51
+ doc.css("a[href^='#']").each do |a|
52
+ status, href = resolve(a["href"].to_s.delete_prefix("#"),
53
+ path, id_index)
54
+ status = :unresolved if status.nil?
55
+ counts[status] += 1
56
+ a["href"] = href if href
57
+ end
58
+ end
59
+
60
+ # [status, href-or-nil]: :own = the target id is on the citing
61
+ # page (fine as written); :ambiguous = several candidates;
62
+ # :unresolved = no candidate; :remapped carries the new href.
63
+ def resolve(frag, path, id_index)
64
+ exact = id_index[frag]
65
+ if exact
66
+ uniq = exact[:pages].uniq
67
+ return [:own] if uniq == [path]
68
+ return [:ambiguous] if uniq.size > 1
69
+
70
+ return [:remapped, cross_href(path, uniq.first, frag)]
71
+ end
72
+
73
+ scoped_resolve(frag, path, id_index)
74
+ end
75
+
76
+ def scoped_resolve(frag, path, id_index)
77
+ candidates = matching_scopes(frag, path, id_index)
78
+ return [:ambiguous] if candidates.size > 1
79
+
80
+ id, pages = candidates.first
81
+ return [:unresolved, nil] unless id
82
+
83
+ [:remapped, cross_href(path, pages.first, id)]
84
+ end
85
+
86
+ # [id, pages] pairs for stamped anchors whose scope names the
87
+ # fragment, rendered on some page other than the citing one.
88
+ def matching_scopes(frag, path, id_index)
89
+ id_index.filter_map do |id, entry|
90
+ pages = entry[:pages].uniq
91
+ next if pages == [path]
92
+
93
+ scope = scope_of(id)
94
+ next unless scope == frag || scope.end_with?(".#{frag}")
95
+
96
+ [id, pages]
97
+ end
98
+ end
99
+
100
+ def scope_of(id)
101
+ SCOPED_ID_RE.match(id)&.[](1)
102
+ end
103
+
104
+ def cross_href(path, page, fragment)
105
+ "#{relative_href(path, page)}##{fragment}"
106
+ end
107
+
108
+ def relative_href(from, to)
109
+ from.dirname.join(to.basename).relative_path_from(from.dirname).to_s
110
+ end
111
+
112
+ def html_pages
113
+ @html_pages ||= @site_dir.glob("**/*.html")
114
+ .reject { |p| p.basename.to_s.end_with?(".err.html") }
115
+ .to_h { |p| [p, Leptris::XML.parse_html(p.read)] }
116
+ end
117
+
118
+ # fragment id → { pages: [path, …] }. A plain hash: reads of
119
+ # unknown ids must not insert (matching_scopes iterates the index
120
+ # afterwards).
121
+ def index_ids(pages)
122
+ index = {}
123
+ pages.each do |path, doc|
124
+ doc.css("[id]").each do |el|
125
+ next if (id = el["id"]).nil? || id.empty?
126
+
127
+ (index[id] ||= { pages: [] })[:pages] << path
128
+ end
129
+ end
130
+ index
131
+ end
132
+ end
133
+ end
134
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Suma
4
+ module Delivery
5
+ # Corpus-side gate thresholds for `delivery verify --expectations`:
6
+ # the measured site-shape conditions as data, so the calling workflow
7
+ # states its delivery contract in one manifest instead of hand-rolled
8
+ # shell checks. Unset attributes are not checked.
9
+ class Expectations < Lutaml::Model::Serializable
10
+ attribute :sections_min, :integer
11
+ attribute :index_pages, :integer
12
+ attribute :schema_attachments_min, :integer
13
+ attribute :schema_docs_min, :integer
14
+ attribute :missing_express_max, :integer, default: -> { 0 }
15
+ attribute :broken_links_max, :integer, default: -> { 0 }
16
+
17
+ key_value do
18
+ map "sections_min", to: :sections_min
19
+ map "index_pages", to: :index_pages
20
+ map "schema_attachments_min", to: :schema_attachments_min
21
+ map "schema_docs_min", to: :schema_docs_min
22
+ map "missing_express_max", to: :missing_express_max
23
+ map "broken_links_max", to: :broken_links_max
24
+ end
25
+
26
+ # Human-readable violation lines for every threshold the +inventory+
27
+ # and +broken_links+ breach; unset thresholds stay silent. The name
28
+ # suffix is the comparison: `_min` floors, `_max` caps, a bare name
29
+ # (index_pages) is exact.
30
+ def violations(inventory, broken_links)
31
+ actuals(inventory, broken_links).filter_map do |name, actual|
32
+ limit = public_send(name) or next
33
+ relation = relation_of(name)
34
+ next if relation_met?(relation, actual, limit)
35
+
36
+ format("%<name>s: got %<actual>d, expected %<relation>s " \
37
+ "%<limit>d", name: name, actual: actual,
38
+ relation: relation, limit: limit)
39
+ end
40
+ end
41
+
42
+ private
43
+
44
+ def relation_of(name)
45
+ return "at least" if name.end_with?("_min")
46
+ return "at most" if name.end_with?("_max")
47
+
48
+ "exactly"
49
+ end
50
+
51
+ def relation_met?(relation, actual, limit)
52
+ case relation
53
+ when "at least" then actual >= limit
54
+ when "at most" then actual <= limit
55
+ else actual == limit
56
+ end
57
+ end
58
+
59
+ # Every threshold's observed value, keyed by threshold name.
60
+ def actuals(inventory, broken_links)
61
+ {
62
+ "sections_min" => inventory["sections"],
63
+ "index_pages" => inventory["index_pages"],
64
+ "schema_attachments_min" => inventory["schema_attachments"],
65
+ "schema_docs_min" => inventory["schema_docs"],
66
+ "missing_express_max" => inventory["missing_express_pages"].size,
67
+ "broken_links_max" => broken_links.size,
68
+ }
69
+ end
70
+ end
71
+ end
72
+ end
@@ -18,7 +18,10 @@ module Suma
18
18
  # without making it clearer.
19
19
  # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
20
20
  SCHEME_RE = %r{\A(?:https?:|mailto:|javascript:|data:)}
21
- BARE_INTEGER_RE = /\A\d+\z/
21
+
22
+ # Metanorma's unresolved-citation rendering prefix; a page carrying
23
+ # it cites an express reference the corpus never resolved.
24
+ MISSING_EXPRESS_MARKER = "Missing:express"
22
25
 
23
26
  Report = Struct.new(:report, keyword_init: true) do
24
27
  def broken_count
@@ -40,10 +43,15 @@ module Suma
40
43
  anchor_index = build_anchor_index
41
44
  html_files.each do |html_file|
42
45
  rel_path = html_file.relative_path_from(@site_dir).to_s
43
- doc = Leptris::XML.parse_html(html_file.read)
46
+ content = html_file.read
47
+ if content.include?(MISSING_EXPRESS_MARKER)
48
+ report["missing_express_pages"] << rel_path
49
+ end
50
+ doc = Leptris::XML.parse_html(content)
44
51
  verify_intra_fragments(doc, rel_path, html_file, report)
45
52
  verify_links(doc, rel_path, html_file, anchor_index, report)
46
53
  end
54
+ report["inventory"] = site_inventory
47
55
  Report.new(report: report)
48
56
  end
49
57
 
@@ -61,6 +69,26 @@ module Suma
61
69
  "total_intra_fragments" => 0,
62
70
  "resolved_intra_fragments" => 0,
63
71
  "broken_intra_fragments" => [],
72
+ "missing_express_pages" => [],
73
+ "inventory" => {},
74
+ }
75
+ end
76
+
77
+ # The stitched site's shape: the counts the delivery contract is
78
+ # stated against (section pages, index pages, schema attachments,
79
+ # schema doc pages).
80
+ def site_inventory
81
+ {
82
+ "sections" => Dir.glob("#{@site_dir}/**/member_*.xml.*.html")
83
+ .count { |f| !f.end_with?(".err.html") },
84
+ "index_pages" => Dir.glob("#{@site_dir}/**/*_index.html").size,
85
+ "schema_attachments" =>
86
+ Dir.glob("#{@site_dir}/plain_schemas/**/*.exp").size,
87
+ "schema_docs" => if Dir.exist?("#{@site_dir}/schema_docs")
88
+ Dir.children("#{@site_dir}/schema_docs").size
89
+ else
90
+ 0
91
+ end,
64
92
  }
65
93
  end
66
94
 
@@ -104,7 +132,10 @@ module Suma
104
132
  href = a["href"].to_s.strip
105
133
  next if href.empty? || href.start_with?("#")
106
134
  next if href.match?(SCHEME_RE)
107
- next if href.match?(BARE_INTEGER_RE)
135
+
136
+ # Bare integers are counted, not skipped: an unbound svgmap
137
+ # hotspot emits its ordinal as the href, and a link to a
138
+ # nonexistent file "12" is dead whatever the key form.
108
139
 
109
140
  file_part, fragment = href.split("#", 2)
110
141
  file_part = file_part.delete_prefix("./")
data/lib/suma/delivery.rb CHANGED
@@ -5,6 +5,8 @@ module Suma
5
5
  # per-part sites and resolve their unresolved cross-document references.
6
6
  module Delivery
7
7
  autoload :FrameNav, "suma/delivery/frame_nav"
8
+ autoload :CrossPageFragments, "suma/delivery/cross_page_fragments"
9
+ autoload :Expectations, "suma/delivery/expectations"
8
10
  autoload :Pack, "suma/delivery/pack"
9
11
  autoload :Plan, "suma/delivery/plan"
10
12
  autoload :SiteVerifier, "suma/delivery/site_verifier"
data/lib/suma/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Suma
4
- VERSION = "0.5.33"
4
+ VERSION = "0.5.35"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: suma
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.33
4
+ version: 0.5.35
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
@@ -242,6 +242,8 @@ files:
242
242
  - lib/suma/collection_manifest.rb
243
243
  - lib/suma/corpus_graph.rb
244
244
  - lib/suma/delivery.rb
245
+ - lib/suma/delivery/cross_page_fragments.rb
246
+ - lib/suma/delivery/expectations.rb
245
247
  - lib/suma/delivery/frame_nav.rb
246
248
  - lib/suma/delivery/pack.rb
247
249
  - lib/suma/delivery/plan.rb