suma 0.5.22 → 0.5.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +74 -53
- data/lib/suma/cli/build.rb +6 -0
- data/lib/suma/cli/core.rb +6 -0
- data/lib/suma/cli/delivery.rb +129 -0
- data/lib/suma/cli.rb +1 -0
- data/lib/suma/delivery/frame_nav.rb +173 -0
- data/lib/suma/delivery/plan.rb +234 -0
- data/lib/suma/delivery/site_verifier.rb +155 -0
- data/lib/suma/delivery/stitcher.rb +487 -0
- data/lib/suma/delivery.rb +12 -0
- data/lib/suma/processor.rb +4 -1
- data/lib/suma/staged_collection_builder.rb +3 -0
- data/lib/suma/thor_ext.rb +1 -1
- data/lib/suma/version.rb +1 -1
- data/lib/suma.rb +1 -0
- data/suma.gemspec +1 -0
- metadata +22 -2
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "json"
|
|
5
|
+
require "open3"
|
|
6
|
+
require "yaml"
|
|
7
|
+
|
|
8
|
+
module Suma
|
|
9
|
+
module Delivery
|
|
10
|
+
# Enumerates the parts of a stitched delivery from the collection
|
|
11
|
+
# manifest and writes plan.json: one entry per docref with the part's
|
|
12
|
+
# identity, input inventory, and content hashes. Drives the
|
|
13
|
+
# per-document stitcher pipeline (TCSC410303-2952 Stage A).
|
|
14
|
+
class Plan
|
|
15
|
+
DEFAULT_COLLECTION = "collection-resources.yml"
|
|
16
|
+
|
|
17
|
+
attr_reader :repo_root, :collection, :out_path
|
|
18
|
+
|
|
19
|
+
def initialize(repo_root:, out:, collection: DEFAULT_COLLECTION)
|
|
20
|
+
@repo_root = Pathname.new(File.expand_path(repo_root))
|
|
21
|
+
@collection = collection
|
|
22
|
+
@out_path = Pathname.new(out)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def generate
|
|
26
|
+
docrefs = manifest_docrefs(load_collection_manifest)
|
|
27
|
+
unless docrefs.any?
|
|
28
|
+
raise "No manifest.docref entries in #{collection_path}"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
parts = plan_parts(docrefs)
|
|
32
|
+
plan = build_plan(parts)
|
|
33
|
+
write_plan(plan)
|
|
34
|
+
plan
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def collection_path
|
|
40
|
+
repo_root + collection
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def load_collection_manifest
|
|
44
|
+
YAML.safe_load_file(collection_path, aliases: true)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def manifest_docrefs(manifest)
|
|
48
|
+
node = manifest.dig("manifest", "docref")
|
|
49
|
+
[node].compact.flatten
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def plan_parts(docrefs)
|
|
53
|
+
docrefs.filter_map do |ref|
|
|
54
|
+
next unless ref.is_a?(Hash) && ref["file"]
|
|
55
|
+
|
|
56
|
+
part_collection = repo_root + ref["file"]
|
|
57
|
+
unless part_collection.exist?
|
|
58
|
+
Utils.log "WARN: docref not found: #{part_collection}"
|
|
59
|
+
next
|
|
60
|
+
end
|
|
61
|
+
analyze_part(part_collection.parent)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def build_plan(parts)
|
|
66
|
+
{
|
|
67
|
+
"version" => 1,
|
|
68
|
+
"source_collection" => collection,
|
|
69
|
+
"build_commit" => build_commit,
|
|
70
|
+
"build_timestamp" => Time.now.utc.iso8601,
|
|
71
|
+
"parts_count" => parts.size,
|
|
72
|
+
"parts" => parts,
|
|
73
|
+
}
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def analyze_part(part_dir)
|
|
77
|
+
coll = YAML.safe_load_file(File.join(part_dir, "collection.yml"),
|
|
78
|
+
aliases: true)
|
|
79
|
+
bibdata = coll["bibdata"] || {}
|
|
80
|
+
doc_id = fetch_doc_id(bibdata, part_dir)
|
|
81
|
+
year = derive_year(part_dir, bibdata)
|
|
82
|
+
exp_files = exp_sources(part_dir, coll)
|
|
83
|
+
inputs = inventory(part_dir) + exp_files
|
|
84
|
+
|
|
85
|
+
part_identity(part_dir, doc_id,
|
|
86
|
+
year).merge(part_hashes(inputs, exp_files))
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def part_identity(part_dir, doc_id, year)
|
|
90
|
+
{
|
|
91
|
+
"part_dir" => part_dir.basename.to_s,
|
|
92
|
+
"source_dir" => part_dir.relative_path_from(repo_root).to_s,
|
|
93
|
+
"iso_name" => "ISO_#{doc_id}_#{year}",
|
|
94
|
+
"doc_id" => doc_id,
|
|
95
|
+
"year" => year,
|
|
96
|
+
}
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def part_hashes(inputs, exp_files)
|
|
100
|
+
{
|
|
101
|
+
"doc_hash" => "sha256:#{sha256_files(inputs)}",
|
|
102
|
+
"deps_hash" => "sha256:#{sha256_files(exp_files)}",
|
|
103
|
+
"input_files" => inputs.map { |p| rel_or_abs(p) },
|
|
104
|
+
"exp_files" => exp_files.map { |p| rel_or_abs(p) },
|
|
105
|
+
}
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def fetch_doc_id(bibdata, part_dir)
|
|
109
|
+
node = bibdata["docid"] || {}
|
|
110
|
+
node = node.first if node.is_a?(Array)
|
|
111
|
+
id = node && node["id"]
|
|
112
|
+
id || part_dir.basename.to_s.delete_prefix("iso-")
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def derive_year(part_dir, bibdata)
|
|
116
|
+
adoc = File.join(part_dir, "document.adoc")
|
|
117
|
+
return copyright_from_bibdata(bibdata) unless File.exist?(adoc)
|
|
118
|
+
|
|
119
|
+
File.foreach(adoc) do |line|
|
|
120
|
+
if (m = line.match(/\A:copyright-year:\s*(\d+)/))
|
|
121
|
+
return m[1]
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
copyright_from_bibdata(bibdata)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def copyright_from_bibdata(bibdata)
|
|
128
|
+
from = bibdata.dig("copyright", "from")
|
|
129
|
+
from ? from.to_s : "0000"
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def inventory(part_dir)
|
|
133
|
+
inputs = [File.join(part_dir, "collection.yml")]
|
|
134
|
+
adoc = File.join(part_dir, "document.adoc")
|
|
135
|
+
inputs << adoc if File.exist?(adoc)
|
|
136
|
+
inputs += section_files(part_dir)
|
|
137
|
+
cover = File.join(part_dir, "cover.html")
|
|
138
|
+
inputs << cover if File.exist?(cover)
|
|
139
|
+
inputs
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def section_files(part_dir)
|
|
143
|
+
sections = File.join(part_dir, "sections")
|
|
144
|
+
return [] unless File.directory?(sections)
|
|
145
|
+
|
|
146
|
+
Dir.glob(File.join(sections, "**/*")).select { |p| File.file?(p) }
|
|
147
|
+
.sort.map { |p| Pathname.new(p) }
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def exp_sources(part_dir, coll)
|
|
151
|
+
entries = walk_docrefs(coll["manifest"] || {})
|
|
152
|
+
entries.filter_map do |entry|
|
|
153
|
+
ref = entry["fileref"]
|
|
154
|
+
next unless ref.is_a?(String) && ref.end_with?(".exp")
|
|
155
|
+
|
|
156
|
+
resolve_exp_source(ref, part_dir)
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def walk_docrefs(node, acc = [])
|
|
161
|
+
return acc unless node.is_a?(Hash)
|
|
162
|
+
|
|
163
|
+
docref = node["docref"]
|
|
164
|
+
[docref].compact.flatten.each do |entry|
|
|
165
|
+
next unless entry.is_a?(Hash)
|
|
166
|
+
|
|
167
|
+
acc << entry
|
|
168
|
+
walk_docrefs(entry, acc)
|
|
169
|
+
end
|
|
170
|
+
acc
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# collection.yml filerefs point at plain_schemas/ (build-time
|
|
174
|
+
# output); the source lives at schemas/ under the same suffix.
|
|
175
|
+
def resolve_exp_source(fileref, part_dir)
|
|
176
|
+
raw = (part_dir + fileref).expand_path
|
|
177
|
+
return raw if raw.exist? && under_repo?(raw)
|
|
178
|
+
|
|
179
|
+
candidate = Pathname.new(raw.to_s.sub("/plain_schemas/", "/schemas/"))
|
|
180
|
+
candidate.exist? ? candidate : nil
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def sha256_files(paths)
|
|
184
|
+
h = Digest::SHA256.new
|
|
185
|
+
paths.each { |path| hash_one_file(h, path) }
|
|
186
|
+
h.hexdigest
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def hash_one_file(hash, path)
|
|
190
|
+
hash.update(rel_or_abs(path))
|
|
191
|
+
hash.update("\0")
|
|
192
|
+
File.open(path, "rb") do |f|
|
|
193
|
+
while (chunk = f.read(65_536))
|
|
194
|
+
hash.update(chunk)
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
hash.update("\0\0")
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def rel_or_abs(path)
|
|
201
|
+
path = Pathname.new(path)
|
|
202
|
+
under_repo?(path) ? path.relative_path_from(repo_root).to_s : path.to_s
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def under_repo?(path)
|
|
206
|
+
path.expand_path.to_s.start_with?("#{repo_root.expand_path}/")
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def build_commit
|
|
210
|
+
@build_commit ||=
|
|
211
|
+
begin
|
|
212
|
+
git("rev-parse", "HEAD")
|
|
213
|
+
rescue StandardError
|
|
214
|
+
"unknown"
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def git(*args)
|
|
219
|
+
out, err, status = Open3.capture3("git", "-C", repo_root.to_s, *args)
|
|
220
|
+
raise "git #{args.first} failed: #{err}" unless status.success?
|
|
221
|
+
|
|
222
|
+
out.strip
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def write_plan(plan)
|
|
226
|
+
out_path.parent.mkpath
|
|
227
|
+
File.write(out_path, JSON.pretty_generate(plan))
|
|
228
|
+
Utils.log "plan.json written: #{out_path} " \
|
|
229
|
+
"(parts: #{plan['parts_count']}, " \
|
|
230
|
+
"commit: #{plan['build_commit']})"
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
end
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "pathname"
|
|
5
|
+
require "leptris"
|
|
6
|
+
|
|
7
|
+
module Suma
|
|
8
|
+
module Delivery
|
|
9
|
+
# Site-wide link verification (port of site_manifest.py verify_links):
|
|
10
|
+
# every cross-file href in the stitched site must resolve to a file,
|
|
11
|
+
# and every fragment must resolve to an anchor in its target file.
|
|
12
|
+
# The production gate is zero broken links. Intra-file fragment hrefs
|
|
13
|
+
# are reported but never fail the build — metanorma emits some whose
|
|
14
|
+
# target is never defined (pre-existing upstream issues, not stitch
|
|
15
|
+
# faults).
|
|
16
|
+
class SiteVerifier
|
|
17
|
+
# A faithful port: the metrics would fragment the link walk
|
|
18
|
+
# without making it clearer.
|
|
19
|
+
# rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
20
|
+
SCHEME_RE = %r{\A(?:https?:|mailto:|javascript:|data:)}
|
|
21
|
+
BARE_INTEGER_RE = /\A\d+\z/
|
|
22
|
+
|
|
23
|
+
Report = Struct.new(:report, keyword_init: true) do
|
|
24
|
+
def broken_count
|
|
25
|
+
report["broken_links"].size + report["broken_fragments"].size
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def to_json(*)
|
|
29
|
+
JSON.generate(report)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def initialize(site_dir, check_fragments: true)
|
|
34
|
+
@site_dir = Pathname.new(site_dir)
|
|
35
|
+
@check_fragments = check_fragments
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def verify
|
|
39
|
+
report = empty_report
|
|
40
|
+
anchor_index = build_anchor_index
|
|
41
|
+
html_files.each do |html_file|
|
|
42
|
+
rel_path = html_file.relative_path_from(@site_dir).to_s
|
|
43
|
+
doc = Leptris::XML.parse_html(html_file.read)
|
|
44
|
+
verify_intra_fragments(doc, rel_path, html_file, report)
|
|
45
|
+
verify_links(doc, rel_path, html_file, anchor_index, report)
|
|
46
|
+
end
|
|
47
|
+
Report.new(report: report)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def empty_report
|
|
53
|
+
{
|
|
54
|
+
"total_links" => 0,
|
|
55
|
+
"resolved_links" => 0,
|
|
56
|
+
"broken_links" => [],
|
|
57
|
+
"total_fragments" => 0,
|
|
58
|
+
"resolved_fragments" => 0,
|
|
59
|
+
"broken_fragments" => [],
|
|
60
|
+
"cross_document_links" => 0,
|
|
61
|
+
"total_intra_fragments" => 0,
|
|
62
|
+
"resolved_intra_fragments" => 0,
|
|
63
|
+
"broken_intra_fragments" => [],
|
|
64
|
+
}
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def html_files
|
|
68
|
+
Dir.glob("#{@site_dir}/**/*.html")
|
|
69
|
+
.reject { |f| f.end_with?(".err.html") }
|
|
70
|
+
.sort.map { |f| Pathname.new(f) }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def anchors_of(path)
|
|
74
|
+
Leptris::XML.parse_html(path.read).css("[id]").map { |n| n["id"] }.uniq
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def build_anchor_index
|
|
78
|
+
html_files.to_h do |html_file|
|
|
79
|
+
rel = html_file.relative_path_from(@site_dir).to_s
|
|
80
|
+
[rel, anchors_of(html_file)]
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def verify_intra_fragments(doc, rel_path, html_file, report)
|
|
85
|
+
own_anchors = anchors_of(html_file)
|
|
86
|
+
doc.css("a[href]").each do |a|
|
|
87
|
+
href = a["href"].to_s.strip
|
|
88
|
+
next unless href.start_with?("#") && href.length > 1
|
|
89
|
+
|
|
90
|
+
fragment = href[1..]
|
|
91
|
+
report["total_intra_fragments"] += 1
|
|
92
|
+
if own_anchors.include?(fragment)
|
|
93
|
+
report["resolved_intra_fragments"] += 1
|
|
94
|
+
else
|
|
95
|
+
report["broken_intra_fragments"] <<
|
|
96
|
+
{ "source" => rel_path, "fragment" => fragment }
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def verify_links(doc, rel_path, html_file, anchor_index, report)
|
|
102
|
+
source_dir = Pathname.new(rel_path).parent
|
|
103
|
+
doc.css("a[href]").each do |a|
|
|
104
|
+
href = a["href"].to_s.strip
|
|
105
|
+
next if href.empty? || href.start_with?("#")
|
|
106
|
+
next if href.match?(SCHEME_RE)
|
|
107
|
+
next if href.match?(BARE_INTEGER_RE)
|
|
108
|
+
|
|
109
|
+
file_part, fragment = href.split("#", 2)
|
|
110
|
+
file_part = file_part.delete_prefix("./")
|
|
111
|
+
|
|
112
|
+
report["total_links"] += 1
|
|
113
|
+
target = html_file.parent / file_part
|
|
114
|
+
unless target.exist?
|
|
115
|
+
report["broken_links"] << {
|
|
116
|
+
"source" => rel_path,
|
|
117
|
+
"href" => fragment ? "#{file_part}##{fragment}" : file_part,
|
|
118
|
+
}
|
|
119
|
+
next
|
|
120
|
+
end
|
|
121
|
+
report["resolved_links"] += 1
|
|
122
|
+
count_cross_document(source_dir, target, report)
|
|
123
|
+
verify_fragment(fragment, target, rel_path, anchor_index, report)
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def count_cross_document(source_dir, target, report)
|
|
128
|
+
target_rel = target.relative_path_from(@site_dir)
|
|
129
|
+
return unless source_dir != target_rel.parent && source_dir.to_s != "."
|
|
130
|
+
|
|
131
|
+
report["cross_document_links"] += 1
|
|
132
|
+
rescue ArgumentError
|
|
133
|
+
nil
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def verify_fragment(fragment, target, rel_path, anchor_index, report)
|
|
137
|
+
return unless fragment && @check_fragments
|
|
138
|
+
|
|
139
|
+
report["total_fragments"] += 1
|
|
140
|
+
target_rel = target.relative_path_from(@site_dir).to_s
|
|
141
|
+
if anchor_index.fetch(target_rel, Set.new).include?(fragment)
|
|
142
|
+
report["resolved_fragments"] += 1
|
|
143
|
+
else
|
|
144
|
+
report["broken_fragments"] << {
|
|
145
|
+
"source" => rel_path,
|
|
146
|
+
"href" => "#{target.basename}##{fragment}",
|
|
147
|
+
"fragment" => fragment,
|
|
148
|
+
"target_file" => target_rel,
|
|
149
|
+
}
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
# rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|