suma 0.5.22 → 0.5.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,234 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require "json"
5
+ require "open3"
6
+ require "yaml"
7
+
8
+ module Suma
9
+ module Delivery
10
+ # Enumerates the parts of a stitched delivery from the collection
11
+ # manifest and writes plan.json: one entry per docref with the part's
12
+ # identity, input inventory, and content hashes. Drives the
13
+ # per-document stitcher pipeline (TCSC410303-2952 Stage A).
14
+ class Plan
15
+ DEFAULT_COLLECTION = "collection-resources.yml"
16
+
17
+ attr_reader :repo_root, :collection, :out_path
18
+
19
+ def initialize(repo_root:, out:, collection: DEFAULT_COLLECTION)
20
+ @repo_root = Pathname.new(File.expand_path(repo_root))
21
+ @collection = collection
22
+ @out_path = Pathname.new(out)
23
+ end
24
+
25
+ def generate
26
+ docrefs = manifest_docrefs(load_collection_manifest)
27
+ unless docrefs.any?
28
+ raise "No manifest.docref entries in #{collection_path}"
29
+ end
30
+
31
+ parts = plan_parts(docrefs)
32
+ plan = build_plan(parts)
33
+ write_plan(plan)
34
+ plan
35
+ end
36
+
37
+ private
38
+
39
+ def collection_path
40
+ repo_root + collection
41
+ end
42
+
43
+ def load_collection_manifest
44
+ YAML.safe_load_file(collection_path, aliases: true)
45
+ end
46
+
47
+ def manifest_docrefs(manifest)
48
+ node = manifest.dig("manifest", "docref")
49
+ [node].compact.flatten
50
+ end
51
+
52
+ def plan_parts(docrefs)
53
+ docrefs.filter_map do |ref|
54
+ next unless ref.is_a?(Hash) && ref["file"]
55
+
56
+ part_collection = repo_root + ref["file"]
57
+ unless part_collection.exist?
58
+ Utils.log "WARN: docref not found: #{part_collection}"
59
+ next
60
+ end
61
+ analyze_part(part_collection.parent)
62
+ end
63
+ end
64
+
65
+ def build_plan(parts)
66
+ {
67
+ "version" => 1,
68
+ "source_collection" => collection,
69
+ "build_commit" => build_commit,
70
+ "build_timestamp" => Time.now.utc.iso8601,
71
+ "parts_count" => parts.size,
72
+ "parts" => parts,
73
+ }
74
+ end
75
+
76
+ def analyze_part(part_dir)
77
+ coll = YAML.safe_load_file(File.join(part_dir, "collection.yml"),
78
+ aliases: true)
79
+ bibdata = coll["bibdata"] || {}
80
+ doc_id = fetch_doc_id(bibdata, part_dir)
81
+ year = derive_year(part_dir, bibdata)
82
+ exp_files = exp_sources(part_dir, coll)
83
+ inputs = inventory(part_dir) + exp_files
84
+
85
+ part_identity(part_dir, doc_id,
86
+ year).merge(part_hashes(inputs, exp_files))
87
+ end
88
+
89
+ def part_identity(part_dir, doc_id, year)
90
+ {
91
+ "part_dir" => part_dir.basename.to_s,
92
+ "source_dir" => part_dir.relative_path_from(repo_root).to_s,
93
+ "iso_name" => "ISO_#{doc_id}_#{year}",
94
+ "doc_id" => doc_id,
95
+ "year" => year,
96
+ }
97
+ end
98
+
99
+ def part_hashes(inputs, exp_files)
100
+ {
101
+ "doc_hash" => "sha256:#{sha256_files(inputs)}",
102
+ "deps_hash" => "sha256:#{sha256_files(exp_files)}",
103
+ "input_files" => inputs.map { |p| rel_or_abs(p) },
104
+ "exp_files" => exp_files.map { |p| rel_or_abs(p) },
105
+ }
106
+ end
107
+
108
+ def fetch_doc_id(bibdata, part_dir)
109
+ node = bibdata["docid"] || {}
110
+ node = node.first if node.is_a?(Array)
111
+ id = node && node["id"]
112
+ id || part_dir.basename.to_s.delete_prefix("iso-")
113
+ end
114
+
115
+ def derive_year(part_dir, bibdata)
116
+ adoc = File.join(part_dir, "document.adoc")
117
+ return copyright_from_bibdata(bibdata) unless File.exist?(adoc)
118
+
119
+ File.foreach(adoc) do |line|
120
+ if (m = line.match(/\A:copyright-year:\s*(\d+)/))
121
+ return m[1]
122
+ end
123
+ end
124
+ copyright_from_bibdata(bibdata)
125
+ end
126
+
127
+ def copyright_from_bibdata(bibdata)
128
+ from = bibdata.dig("copyright", "from")
129
+ from ? from.to_s : "0000"
130
+ end
131
+
132
+ def inventory(part_dir)
133
+ inputs = [File.join(part_dir, "collection.yml")]
134
+ adoc = File.join(part_dir, "document.adoc")
135
+ inputs << adoc if File.exist?(adoc)
136
+ inputs += section_files(part_dir)
137
+ cover = File.join(part_dir, "cover.html")
138
+ inputs << cover if File.exist?(cover)
139
+ inputs
140
+ end
141
+
142
+ def section_files(part_dir)
143
+ sections = File.join(part_dir, "sections")
144
+ return [] unless File.directory?(sections)
145
+
146
+ Dir.glob(File.join(sections, "**/*")).select { |p| File.file?(p) }
147
+ .sort.map { |p| Pathname.new(p) }
148
+ end
149
+
150
+ def exp_sources(part_dir, coll)
151
+ entries = walk_docrefs(coll["manifest"] || {})
152
+ entries.filter_map do |entry|
153
+ ref = entry["fileref"]
154
+ next unless ref.is_a?(String) && ref.end_with?(".exp")
155
+
156
+ resolve_exp_source(ref, part_dir)
157
+ end
158
+ end
159
+
160
+ def walk_docrefs(node, acc = [])
161
+ return acc unless node.is_a?(Hash)
162
+
163
+ docref = node["docref"]
164
+ [docref].compact.flatten.each do |entry|
165
+ next unless entry.is_a?(Hash)
166
+
167
+ acc << entry
168
+ walk_docrefs(entry, acc)
169
+ end
170
+ acc
171
+ end
172
+
173
+ # collection.yml filerefs point at plain_schemas/ (build-time
174
+ # output); the source lives at schemas/ under the same suffix.
175
+ def resolve_exp_source(fileref, part_dir)
176
+ raw = (part_dir + fileref).expand_path
177
+ return raw if raw.exist? && under_repo?(raw)
178
+
179
+ candidate = Pathname.new(raw.to_s.sub("/plain_schemas/", "/schemas/"))
180
+ candidate.exist? ? candidate : nil
181
+ end
182
+
183
+ def sha256_files(paths)
184
+ h = Digest::SHA256.new
185
+ paths.each { |path| hash_one_file(h, path) }
186
+ h.hexdigest
187
+ end
188
+
189
+ def hash_one_file(hash, path)
190
+ hash.update(rel_or_abs(path))
191
+ hash.update("\0")
192
+ File.open(path, "rb") do |f|
193
+ while (chunk = f.read(65_536))
194
+ hash.update(chunk)
195
+ end
196
+ end
197
+ hash.update("\0\0")
198
+ end
199
+
200
+ def rel_or_abs(path)
201
+ path = Pathname.new(path)
202
+ under_repo?(path) ? path.relative_path_from(repo_root).to_s : path.to_s
203
+ end
204
+
205
+ def under_repo?(path)
206
+ path.expand_path.to_s.start_with?("#{repo_root.expand_path}/")
207
+ end
208
+
209
+ def build_commit
210
+ @build_commit ||=
211
+ begin
212
+ git("rev-parse", "HEAD")
213
+ rescue StandardError
214
+ "unknown"
215
+ end
216
+ end
217
+
218
+ def git(*args)
219
+ out, err, status = Open3.capture3("git", "-C", repo_root.to_s, *args)
220
+ raise "git #{args.first} failed: #{err}" unless status.success?
221
+
222
+ out.strip
223
+ end
224
+
225
+ def write_plan(plan)
226
+ out_path.parent.mkpath
227
+ File.write(out_path, JSON.pretty_generate(plan))
228
+ Utils.log "plan.json written: #{out_path} " \
229
+ "(parts: #{plan['parts_count']}, " \
230
+ "commit: #{plan['build_commit']})"
231
+ end
232
+ end
233
+ end
234
+ end
@@ -0,0 +1,155 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "pathname"
5
+ require "leptris"
6
+
7
+ module Suma
8
+ module Delivery
9
+ # Site-wide link verification (port of site_manifest.py verify_links):
10
+ # every cross-file href in the stitched site must resolve to a file,
11
+ # and every fragment must resolve to an anchor in its target file.
12
+ # The production gate is zero broken links. Intra-file fragment hrefs
13
+ # are reported but never fail the build — metanorma emits some whose
14
+ # target is never defined (pre-existing upstream issues, not stitch
15
+ # faults).
16
+ class SiteVerifier
17
+ # A faithful port: the metrics would fragment the link walk
18
+ # without making it clearer.
19
+ # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
20
+ SCHEME_RE = %r{\A(?:https?:|mailto:|javascript:|data:)}
21
+ BARE_INTEGER_RE = /\A\d+\z/
22
+
23
+ Report = Struct.new(:report, keyword_init: true) do
24
+ def broken_count
25
+ report["broken_links"].size + report["broken_fragments"].size
26
+ end
27
+
28
+ def to_json(*)
29
+ JSON.generate(report)
30
+ end
31
+ end
32
+
33
+ def initialize(site_dir, check_fragments: true)
34
+ @site_dir = Pathname.new(site_dir)
35
+ @check_fragments = check_fragments
36
+ end
37
+
38
+ def verify
39
+ report = empty_report
40
+ anchor_index = build_anchor_index
41
+ html_files.each do |html_file|
42
+ rel_path = html_file.relative_path_from(@site_dir).to_s
43
+ doc = Leptris::XML.parse_html(html_file.read)
44
+ verify_intra_fragments(doc, rel_path, html_file, report)
45
+ verify_links(doc, rel_path, html_file, anchor_index, report)
46
+ end
47
+ Report.new(report: report)
48
+ end
49
+
50
+ private
51
+
52
+ def empty_report
53
+ {
54
+ "total_links" => 0,
55
+ "resolved_links" => 0,
56
+ "broken_links" => [],
57
+ "total_fragments" => 0,
58
+ "resolved_fragments" => 0,
59
+ "broken_fragments" => [],
60
+ "cross_document_links" => 0,
61
+ "total_intra_fragments" => 0,
62
+ "resolved_intra_fragments" => 0,
63
+ "broken_intra_fragments" => [],
64
+ }
65
+ end
66
+
67
+ def html_files
68
+ Dir.glob("#{@site_dir}/**/*.html")
69
+ .reject { |f| f.end_with?(".err.html") }
70
+ .sort.map { |f| Pathname.new(f) }
71
+ end
72
+
73
+ def anchors_of(path)
74
+ Leptris::XML.parse_html(path.read).css("[id]").map { |n| n["id"] }.uniq
75
+ end
76
+
77
+ def build_anchor_index
78
+ html_files.to_h do |html_file|
79
+ rel = html_file.relative_path_from(@site_dir).to_s
80
+ [rel, anchors_of(html_file)]
81
+ end
82
+ end
83
+
84
+ def verify_intra_fragments(doc, rel_path, html_file, report)
85
+ own_anchors = anchors_of(html_file)
86
+ doc.css("a[href]").each do |a|
87
+ href = a["href"].to_s.strip
88
+ next unless href.start_with?("#") && href.length > 1
89
+
90
+ fragment = href[1..]
91
+ report["total_intra_fragments"] += 1
92
+ if own_anchors.include?(fragment)
93
+ report["resolved_intra_fragments"] += 1
94
+ else
95
+ report["broken_intra_fragments"] <<
96
+ { "source" => rel_path, "fragment" => fragment }
97
+ end
98
+ end
99
+ end
100
+
101
+ def verify_links(doc, rel_path, html_file, anchor_index, report)
102
+ source_dir = Pathname.new(rel_path).parent
103
+ doc.css("a[href]").each do |a|
104
+ href = a["href"].to_s.strip
105
+ next if href.empty? || href.start_with?("#")
106
+ next if href.match?(SCHEME_RE)
107
+ next if href.match?(BARE_INTEGER_RE)
108
+
109
+ file_part, fragment = href.split("#", 2)
110
+ file_part = file_part.delete_prefix("./")
111
+
112
+ report["total_links"] += 1
113
+ target = html_file.parent / file_part
114
+ unless target.exist?
115
+ report["broken_links"] << {
116
+ "source" => rel_path,
117
+ "href" => fragment ? "#{file_part}##{fragment}" : file_part,
118
+ }
119
+ next
120
+ end
121
+ report["resolved_links"] += 1
122
+ count_cross_document(source_dir, target, report)
123
+ verify_fragment(fragment, target, rel_path, anchor_index, report)
124
+ end
125
+ end
126
+
127
+ def count_cross_document(source_dir, target, report)
128
+ target_rel = target.relative_path_from(@site_dir)
129
+ return unless source_dir != target_rel.parent && source_dir.to_s != "."
130
+
131
+ report["cross_document_links"] += 1
132
+ rescue ArgumentError
133
+ nil
134
+ end
135
+
136
+ def verify_fragment(fragment, target, rel_path, anchor_index, report)
137
+ return unless fragment && @check_fragments
138
+
139
+ report["total_fragments"] += 1
140
+ target_rel = target.relative_path_from(@site_dir).to_s
141
+ if anchor_index.fetch(target_rel, Set.new).include?(fragment)
142
+ report["resolved_fragments"] += 1
143
+ else
144
+ report["broken_fragments"] << {
145
+ "source" => rel_path,
146
+ "href" => "#{target.basename}##{fragment}",
147
+ "fragment" => fragment,
148
+ "target_file" => target_rel,
149
+ }
150
+ end
151
+ end
152
+ # rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
153
+ end
154
+ end
155
+ end