suma 0.5.25 → 0.5.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +5 -0
- data/lib/suma/cli/delivery.rb +31 -0
- data/lib/suma/delivery/pack.rb +463 -0
- data/lib/suma/delivery/plan.rb +84 -47
- data/lib/suma/delivery.rb +1 -0
- data/lib/suma/term_extractor.rb +4 -1
- data/lib/suma/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b74619fc760b15ff91e817419ca8153c5d44fc4585867fc71b313f8abf4b9319
|
|
4
|
+
data.tar.gz: ef1be319fc5a2ececeedc657acc8b45af4cdca557708b9949eb0c419b04f4396
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ab67208b4c286f162dda99822685a2bfc550c06690ee0afb9931f2e7e342d3291881020e114483228af3206bd0a8003157366bafc37ea8e3b494c47077b2f29a
|
|
7
|
+
data.tar.gz: 96dc56715bc10acd63d12cd446b0ddf217ac7d53b64fb05570176300846b9404a8bf84e19bcb59cd46a9a082c23c337cf5966b477be5ff79c45aa86db1e64696
|
data/.rubocop_todo.yml
CHANGED
|
@@ -92,6 +92,7 @@ Layout/LineLength:
|
|
|
92
92
|
- 'spec/suma/delivery/site_verifier_spec.rb'
|
|
93
93
|
- 'spec/suma/delivery/stitcher_spec.rb'
|
|
94
94
|
- 'spec/suma/delivery/plan_spec.rb'
|
|
95
|
+
- 'spec/suma/delivery/pack_spec.rb'
|
|
95
96
|
|
|
96
97
|
# Offense count: 2
|
|
97
98
|
# Configuration parameters: IgnoreLiteralBranches, IgnoreConstantBranches, IgnoreDuplicateElseBranch.
|
|
@@ -150,6 +151,7 @@ Metrics/AbcSize:
|
|
|
150
151
|
- 'spec/suma/cli/build_integration_spec.rb'
|
|
151
152
|
- 'spec/suma/reproducibility_spec.rb'
|
|
152
153
|
- spec/suma/delivery/stitcher_spec.rb
|
|
154
|
+
- spec/suma/delivery/pack_spec.rb
|
|
153
155
|
|
|
154
156
|
# Offense count: 2
|
|
155
157
|
# Configuration parameters: CountComments, Max, CountAsOne, AllowedMethods, AllowedPatterns, inherit_mode.
|
|
@@ -338,6 +340,7 @@ RSpec/ExampleLength:
|
|
|
338
340
|
- 'spec/suma/delivery/site_verifier_spec.rb'
|
|
339
341
|
- 'spec/suma/delivery/stitcher_spec.rb'
|
|
340
342
|
- 'spec/suma/delivery/plan_spec.rb'
|
|
343
|
+
- 'spec/suma/delivery/pack_spec.rb'
|
|
341
344
|
|
|
342
345
|
# Offense count: 4
|
|
343
346
|
# Configuration parameters: Max, AllowedIdentifiers, AllowedPatterns.
|
|
@@ -353,6 +356,7 @@ RSpec/InstanceVariable:
|
|
|
353
356
|
Exclude:
|
|
354
357
|
- 'spec/suma/schema_comparer_integration_spec.rb'
|
|
355
358
|
- 'spec/suma/delivery/plan_spec.rb'
|
|
359
|
+
- 'spec/suma/delivery/pack_spec.rb'
|
|
356
360
|
|
|
357
361
|
# Offense count: 1
|
|
358
362
|
# This cop supports safe autocorrection (--autocorrect).
|
|
@@ -430,6 +434,7 @@ RSpec/MultipleExpectations:
|
|
|
430
434
|
- 'spec/suma/delivery/site_verifier_spec.rb'
|
|
431
435
|
- 'spec/suma/delivery/stitcher_spec.rb'
|
|
432
436
|
- 'spec/suma/delivery/plan_spec.rb'
|
|
437
|
+
- 'spec/suma/delivery/pack_spec.rb'
|
|
433
438
|
|
|
434
439
|
# Offense count: 3
|
|
435
440
|
# Configuration parameters: Max, AllowedGroups.
|
data/lib/suma/cli/delivery.rb
CHANGED
|
@@ -75,6 +75,27 @@ module Suma
|
|
|
75
75
|
"from #{result['source_collection']}"
|
|
76
76
|
end
|
|
77
77
|
|
|
78
|
+
desc "pack PART", "Capture a part's built _site/ into a " \
|
|
79
|
+
"stitcher-ready bundle: normalize the per-doc " \
|
|
80
|
+
"layout, scan anchors and unresolved-eref spans, " \
|
|
81
|
+
"copy the owned schema attachments, and write " \
|
|
82
|
+
"<srl-build-base>/parts/<part>/bundle.json"
|
|
83
|
+
option :srl_build_base, type: :string,
|
|
84
|
+
desc: "SRL build base holding plan.json and " \
|
|
85
|
+
"wt/<part>/_site (default: " \
|
|
86
|
+
"$SRL_BUILD_BASE or ~/work/srl-build)"
|
|
87
|
+
option :repo_root, type: :string, default: ".",
|
|
88
|
+
desc: "Repository root holding the part's " \
|
|
89
|
+
"collection.yml"
|
|
90
|
+
option :skip_layout, type: :boolean, default: false,
|
|
91
|
+
desc: "Skip layout normalization (assume " \
|
|
92
|
+
"already done)"
|
|
93
|
+
|
|
94
|
+
def pack(part)
|
|
95
|
+
result = Suma::Delivery::Pack.new(**pack_options(part)).pack
|
|
96
|
+
Utils.log "Packed #{part}: #{result.iso_name} into #{result.bundle_dir}"
|
|
97
|
+
end
|
|
98
|
+
|
|
78
99
|
desc "frame-nav SITE_DIR", "Post-process a stitched site for " \
|
|
79
100
|
"frame navigation: strip the index chrome, " \
|
|
80
101
|
"build the top-level index, wire onclick " \
|
|
@@ -124,6 +145,16 @@ module Suma
|
|
|
124
145
|
out: options[:out],
|
|
125
146
|
}
|
|
126
147
|
end
|
|
148
|
+
|
|
149
|
+
def pack_options(part)
|
|
150
|
+
{
|
|
151
|
+
part: part,
|
|
152
|
+
srl_build_base: options[:srl_build_base] || ENV["SRL_BUILD_BASE"] ||
|
|
153
|
+
File.expand_path("~/work/srl-build"),
|
|
154
|
+
repo_root: options[:repo_root],
|
|
155
|
+
skip_layout: options[:skip_layout],
|
|
156
|
+
}
|
|
157
|
+
end
|
|
127
158
|
end
|
|
128
159
|
end
|
|
129
160
|
end
|
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "json"
|
|
5
|
+
require "pathname"
|
|
6
|
+
require "time"
|
|
7
|
+
require "yaml"
|
|
8
|
+
|
|
9
|
+
require "leptris"
|
|
10
|
+
|
|
11
|
+
module Suma
|
|
12
|
+
module Delivery
|
|
13
|
+
# Capture a part's independently rendered _site/ into a
|
|
14
|
+
# stitcher-ready bundle (TCSC410303-2952 Stage B; port of
|
|
15
|
+
# scripts/srl-build/srl_pack_bundle.py, fusing the iso_renamer.py +
|
|
16
|
+
# align_parts.py layout normalization into one idempotent pass):
|
|
17
|
+
#
|
|
18
|
+
# 1. Normalize the flat document.xml.<N> render into a per-doc
|
|
19
|
+
# directory (<docid>_section_<N>.html …), rewriting intra-site
|
|
20
|
+
# hrefs to the new layout.
|
|
21
|
+
# 2. Scan anchors (every id attribute) and unresolved-eref spans
|
|
22
|
+
# through Leptris.
|
|
23
|
+
# 3. Determine the schemas the part owns from its collection.yml.
|
|
24
|
+
# 4. Copy the per-doc dir, owned schema_docs/ dirs, and
|
|
25
|
+
# plain_schemas/ into <base>/parts/<part>/site/.
|
|
26
|
+
# 5. Write <base>/parts/<part>/bundle.json for the stitcher.
|
|
27
|
+
#
|
|
28
|
+
# The per-doc directory name is parsed from the docid the corpus's
|
|
29
|
+
# own bibdata declares (id tokens joined with underscores, prefixed
|
|
30
|
+
# with the upcased docid type when the id carries no alphabetic
|
|
31
|
+
# prefix token) — e.g. {type: "iso", id: "10303-41"} with year 2025
|
|
32
|
+
# becomes ISO_10303-41_2025; {id: "DIS 10303-104"} becomes
|
|
33
|
+
# DIS_10303-104_<year>. No stage heuristics: whatever the bibdata
|
|
34
|
+
# says is used as-is.
|
|
35
|
+
class Pack
|
|
36
|
+
# The collection-processed suffix bibitemids carry
|
|
37
|
+
# (`_ISO_<doc>_<doc>_<section>`); stripped before schema/entity
|
|
38
|
+
# parsing.
|
|
39
|
+
BIBITEMID_SUFFIX_RE = /_ISO_[A-Za-z0-9_-]+_ISO_[A-Za-z0-9_-]+_.+\z/
|
|
40
|
+
BARE_BIBITEMID_RE = /\Aexpress_([a-z][a-z0-9_]*?)(?:\.(.+))?\z/
|
|
41
|
+
SECTION_FILE_RE = /\Adocument(?:\.\d+)?\.xml\.(\d+)\.html\z/
|
|
42
|
+
SECTION_ERR_FILE_RE = /\Adocument(?:\.\d+)?\.xml\.(\d+)\.err\.html\z/
|
|
43
|
+
|
|
44
|
+
# align_parts.py's schema-link fixups, ported verbatim: every
|
|
45
|
+
# schema_docs/plain_schemas href needs one more `../` once the
|
|
46
|
+
# referencing file moves from _site/ into the per-doc directory
|
|
47
|
+
# (the `../../` double-up fix plus the no-prefix forms).
|
|
48
|
+
SCHEMA_LINK_FIXUPS = [
|
|
49
|
+
[%r{href="\.\./\.\./schema_docs/([^/]+)/([^"]+)"},
|
|
50
|
+
'href="../schema_docs/\1/\2"'],
|
|
51
|
+
[%r{href="\.\./\.\./plain_schemas/(resources|modules)/([^/]+)/([^"]+)"},
|
|
52
|
+
'href="../plain_schemas/\1/\2/\3"'],
|
|
53
|
+
[%r{href="schema_docs/([^/]+)/([^"]+)"}, 'href="../schema_docs/\1/\2"'],
|
|
54
|
+
[%r{href="plain_schemas/(resources|modules)/([^/]+)/([^"]+)"},
|
|
55
|
+
'href="../plain_schemas/\1/\2/\3"'],
|
|
56
|
+
].freeze
|
|
57
|
+
|
|
58
|
+
Result = Struct.new(:bundle_dir, :iso_name, keyword_init: true)
|
|
59
|
+
|
|
60
|
+
attr_reader :base, :repo_root
|
|
61
|
+
|
|
62
|
+
# A faithful port of an algorithmic pipeline: the metrics would
|
|
63
|
+
# fragment it without making it clearer.
|
|
64
|
+
# rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/ParameterLists, Layout/LineLength
|
|
65
|
+
def initialize(part:, srl_build_base:, repo_root:, skip_layout: false)
|
|
66
|
+
@part = part
|
|
67
|
+
@base = Pathname.new(File.expand_path(srl_build_base))
|
|
68
|
+
@repo_root = Pathname.new(File.expand_path(repo_root))
|
|
69
|
+
@skip_layout = skip_layout
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def pack
|
|
73
|
+
plan = load_plan
|
|
74
|
+
entry = plan_entry(plan)
|
|
75
|
+
site = check_worktree_site
|
|
76
|
+
coll_path = part_collection_path(entry)
|
|
77
|
+
identity = Plan.identity(coll_path.parent)
|
|
78
|
+
|
|
79
|
+
Utils.log "Packing bundle for #{@part} (expected #{entry['iso_name']})"
|
|
80
|
+
iso_dir = ensure_layout(site, identity)
|
|
81
|
+
iso_name = iso_dir.basename.to_s
|
|
82
|
+
note_iso_name_mismatch(entry, iso_name)
|
|
83
|
+
|
|
84
|
+
anchors, unresolved_xrefs, sections = scan_sections(iso_dir)
|
|
85
|
+
Utils.log " sections: #{sections.size}"
|
|
86
|
+
Utils.log " anchors (total): #{anchors.values.sum(&:size)}"
|
|
87
|
+
Utils.log " unresolved xrefs: #{unresolved_xrefs.size}"
|
|
88
|
+
|
|
89
|
+
schema_docs, plain_schema_files = owned_attachments(coll_path)
|
|
90
|
+
Utils.log " schemas attached: #{schema_docs.size}"
|
|
91
|
+
|
|
92
|
+
bundle_dir = copy_artifacts(site, iso_dir, schema_docs)
|
|
93
|
+
write_bundle(bundle_dir, plan, entry, identity, iso_name, sections,
|
|
94
|
+
schema_docs, plain_schema_files, anchors, unresolved_xrefs)
|
|
95
|
+
Utils.log "Bundle ready: #{bundle_dir}"
|
|
96
|
+
Result.new(bundle_dir: bundle_dir, iso_name: iso_name)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
# -- inputs -------------------------------------------------------
|
|
102
|
+
|
|
103
|
+
def plan_path
|
|
104
|
+
base / "plan.json"
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def load_plan
|
|
108
|
+
raise "Plan not found: #{plan_path} (run `suma delivery plan` first)" unless
|
|
109
|
+
plan_path.exist?
|
|
110
|
+
|
|
111
|
+
JSON.parse(plan_path.read)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def plan_entry(plan)
|
|
115
|
+
entry = plan["parts"].find { |p| p["part_dir"] == @part }
|
|
116
|
+
raise "Part '#{@part}' not in plan (#{plan_path})" unless entry
|
|
117
|
+
|
|
118
|
+
entry
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def worktree_site
|
|
122
|
+
base / "wt" / @part / "_site"
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def check_worktree_site
|
|
126
|
+
raise "Worktree not found: #{worktree_site.parent}" unless
|
|
127
|
+
worktree_site.parent.exist?
|
|
128
|
+
raise "_site/ not found in #{worktree_site.parent} (build failed?)" unless
|
|
129
|
+
worktree_site.exist?
|
|
130
|
+
|
|
131
|
+
worktree_site
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def part_collection_path(entry)
|
|
135
|
+
repo_root / entry["source_dir"] / "collection.yml"
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# -- Step A: layout normalization ---------------------------------
|
|
139
|
+
|
|
140
|
+
def ensure_layout(site, identity)
|
|
141
|
+
rendered = rendered_doc_name(identity)
|
|
142
|
+
if (existing = find_part_subdir(site, identity, rendered))
|
|
143
|
+
Utils.log "[skip] layout normalization already done"
|
|
144
|
+
return existing
|
|
145
|
+
end
|
|
146
|
+
raise no_subdir_error(site) if @skip_layout
|
|
147
|
+
|
|
148
|
+
mapping = build_file_mapping(site, rendered)
|
|
149
|
+
rewrite_hrefs(site, mapping)
|
|
150
|
+
move_files(site, mapping, site / rendered)
|
|
151
|
+
apply_schema_fixups_dir(site / rendered)
|
|
152
|
+
|
|
153
|
+
find_part_subdir(site, identity, rendered) || raise(no_subdir_error(site))
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def no_subdir_error(site)
|
|
157
|
+
"No <PREFIX>_<docid>_<year>/ subdir found in #{site} for #{@part}"
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# The per-doc directory name, parsed from the bibdata-declared
|
|
161
|
+
# docid: id tokens joined with underscores; when the id carries
|
|
162
|
+
# no alphabetic prefix token, the upcased docid type prefixes it.
|
|
163
|
+
def rendered_doc_name(identity)
|
|
164
|
+
doc_id = identity["doc_id"].to_s
|
|
165
|
+
tokens = doc_id.split
|
|
166
|
+
body =
|
|
167
|
+
if tokens.first&.match?(/\A[A-Za-z]+\z/)
|
|
168
|
+
[tokens.first.upcase, *tokens.drop(1)].join("_")
|
|
169
|
+
else
|
|
170
|
+
[identity["docid_type"].to_s.upcase, doc_id].reject(&:empty?).join("_")
|
|
171
|
+
end
|
|
172
|
+
"#{body}_#{identity['year']}"
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def find_part_subdir(site, identity, rendered)
|
|
176
|
+
return nil unless site.exist?
|
|
177
|
+
|
|
178
|
+
doc_key = identity["doc_id"].to_s.gsub(/\s+/, "_")
|
|
179
|
+
year = identity["year"].to_s
|
|
180
|
+
pattern = /\A[A-Z]+_#{Regexp.escape(doc_key)}_#{Regexp.escape(year)}\z/
|
|
181
|
+
site.children.find do |child|
|
|
182
|
+
child.directory? &&
|
|
183
|
+
(pattern.match?(child.basename.to_s) ||
|
|
184
|
+
child.basename.to_s == rendered)
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def build_file_mapping(site, rendered)
|
|
189
|
+
mapping = {}
|
|
190
|
+
site.children.sort.each do |file|
|
|
191
|
+
next unless file.file?
|
|
192
|
+
|
|
193
|
+
add_section_mapping(mapping, file.basename.to_s, rendered)
|
|
194
|
+
end
|
|
195
|
+
add_index_mapping(mapping, doc_index_file(site), rendered)
|
|
196
|
+
mapping
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# iso_renamer.py PHASE 2 naming: document.xml.<N>.html sections
|
|
200
|
+
# and their .err.html siblings become
|
|
201
|
+
# <rendered>_section_<N>[.err].html. The .xml and
|
|
202
|
+
# .presentation.xml siblings stay at the site root (the Python
|
|
203
|
+
# renamer touches *.html only).
|
|
204
|
+
def add_section_mapping(mapping, name, rendered)
|
|
205
|
+
if (m = SECTION_FILE_RE.match(name))
|
|
206
|
+
mapping[name] = "#{rendered}_section_#{m[1]}.html"
|
|
207
|
+
elsif (m = SECTION_ERR_FILE_RE.match(name))
|
|
208
|
+
mapping[name] = "#{rendered}_section_#{m[1]}.err.html"
|
|
209
|
+
end
|
|
210
|
+
mapping
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def add_index_mapping(mapping, index, rendered)
|
|
214
|
+
return mapping unless index
|
|
215
|
+
|
|
216
|
+
name = index.basename.to_s
|
|
217
|
+
mapping[name] = "#{rendered}_index.html" unless name == "#{rendered}_index.html"
|
|
218
|
+
mapping
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# The top-level *_index.html produced by the single-document
|
|
222
|
+
# render; renamed to <rendered>_index.html for the stitcher's
|
|
223
|
+
# merged index and doc-level links.
|
|
224
|
+
def doc_index_file(site)
|
|
225
|
+
candidates = site.children.select do |child|
|
|
226
|
+
child.file? && child.basename.to_s.end_with?("_index.html")
|
|
227
|
+
end
|
|
228
|
+
if candidates.size > 1
|
|
229
|
+
names = candidates.map { |c| c.basename.to_s }.join(", ")
|
|
230
|
+
raise "Multiple per-doc index files in #{site}: #{names}"
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
candidates.first
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# iso_renamer.py PHASE 3 ported generically: swap every old
|
|
237
|
+
# document/index filename href for its new name, preserving the
|
|
238
|
+
# optional `./` prefix, any `#fragment`, and the quote style.
|
|
239
|
+
def swap_hrefs(content, mapping)
|
|
240
|
+
mapping.each do |old, new|
|
|
241
|
+
content = content.gsub(
|
|
242
|
+
/(href=["'])(\.\/)?(#{Regexp.escape(old)})(#[^"']*)?(["'])/,
|
|
243
|
+
) do
|
|
244
|
+
m = Regexp.last_match
|
|
245
|
+
"#{m[1]}#{m[2]}#{new}#{m[4]}#{m[5]}"
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
content
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
def apply_schema_fixups(content)
|
|
252
|
+
SCHEMA_LINK_FIXUPS.each do |pattern, replacement|
|
|
253
|
+
content = content.gsub(pattern, replacement)
|
|
254
|
+
end
|
|
255
|
+
content
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# Rewrites run Python-ordered: filename swaps over the flat top
|
|
259
|
+
# level and schema_docs/ (iso_renamer PHASE 3), the files move
|
|
260
|
+
# into the per-doc dir (PHASE 4), then align_parts.py's
|
|
261
|
+
# schema-link fixups run on the moved files from their new depth.
|
|
262
|
+
def rewrite_hrefs(site, mapping)
|
|
263
|
+
swap_targets = (site.children.select do |c|
|
|
264
|
+
c.file? && c.basename.to_s.end_with?(".html")
|
|
265
|
+
end + Dir.glob("#{site}/schema_docs/**/*.html")
|
|
266
|
+
.map { |p| Pathname.new(p) }).uniq.sort
|
|
267
|
+
swap_targets.each do |file|
|
|
268
|
+
rewrite_file(file) { |content| swap_hrefs(content, mapping) }
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
def move_files(site, mapping, target_dir)
|
|
273
|
+
FileUtils.mkdir_p(target_dir)
|
|
274
|
+
mapping.each do |old, new|
|
|
275
|
+
src = site / old
|
|
276
|
+
FileUtils.mv(src, target_dir / new) if src.exist?
|
|
277
|
+
end
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
def apply_schema_fixups_dir(dir)
|
|
281
|
+
return unless dir.exist?
|
|
282
|
+
|
|
283
|
+
dir.children.select { |c| c.file? && c.basename.to_s.end_with?(".html") }
|
|
284
|
+
.sort.each do |file|
|
|
285
|
+
rewrite_file(file) { |content| apply_schema_fixups(content) }
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
def rewrite_file(file)
|
|
290
|
+
content = file.read
|
|
291
|
+
rewritten = yield content
|
|
292
|
+
file.write(rewritten) if rewritten != content
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
# -- Step B: scan -------------------------------------------------
|
|
296
|
+
|
|
297
|
+
def scan_sections(iso_dir)
|
|
298
|
+
anchors = {}
|
|
299
|
+
unresolved_xrefs = []
|
|
300
|
+
sections = []
|
|
301
|
+
html_files(iso_dir).each do |html_file|
|
|
302
|
+
name = html_file.basename.to_s
|
|
303
|
+
next if name.end_with?(".err.html")
|
|
304
|
+
|
|
305
|
+
sections << name
|
|
306
|
+
scan_html_file(html_file, name, anchors, unresolved_xrefs)
|
|
307
|
+
end
|
|
308
|
+
[anchors, unresolved_xrefs, sections]
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
def html_files(dir)
|
|
312
|
+
dir.children.select { |c| c.file? && c.basename.to_s.end_with?(".html") }
|
|
313
|
+
.sort
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def scan_html_file(html_file, name, anchors, unresolved_xrefs)
|
|
317
|
+
doc = Leptris::XML.parse_html(html_file.read)
|
|
318
|
+
anchors[name] = doc.css("[id]").filter_map { |el| el["id"] }
|
|
319
|
+
.reject(&:empty?).uniq.sort
|
|
320
|
+
doc.css("span.unresolved-eref").each do |span|
|
|
321
|
+
bibitemid = span["data-bibitemid"].to_s
|
|
322
|
+
next if bibitemid.empty?
|
|
323
|
+
|
|
324
|
+
schema, entity = parse_bibitemid(bibitemid)
|
|
325
|
+
unresolved_xrefs << {
|
|
326
|
+
"source_file" => name,
|
|
327
|
+
"bibitemid" => bibitemid,
|
|
328
|
+
"schema_ref" => schema,
|
|
329
|
+
"entity_ref" => entity,
|
|
330
|
+
"display_text" => span.text.strip,
|
|
331
|
+
"citeas" => span["data-citeas"].to_s,
|
|
332
|
+
}
|
|
333
|
+
end
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
# srl_pack_bundle.py#parse_bibitemid: strip the collection-
|
|
337
|
+
# processed `_ISO_<doc>_<doc>_<section>` suffix, then split an
|
|
338
|
+
# `express_<schema>[.<entity>]` bibitemid.
|
|
339
|
+
def parse_bibitemid(bibitemid)
|
|
340
|
+
bare = bibitemid.gsub(BIBITEMID_SUFFIX_RE, "")
|
|
341
|
+
m = BARE_BIBITEMID_RE.match(bare)
|
|
342
|
+
m ? [m[1], m[2]] : [nil, nil]
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
# -- Step C: owned schema attachments -----------------------------
|
|
346
|
+
|
|
347
|
+
def owned_attachments(coll_path)
|
|
348
|
+
coll = YAML.safe_load_file(coll_path, aliases: true)
|
|
349
|
+
part_dir = coll_path.parent
|
|
350
|
+
schema_docs = []
|
|
351
|
+
plain_schema_files = []
|
|
352
|
+
Plan.walk_docrefs(coll["manifest"] || {}).each do |docref|
|
|
353
|
+
ref = docref["fileref"]
|
|
354
|
+
next unless ref.is_a?(String)
|
|
355
|
+
|
|
356
|
+
collect_attachment(ref, part_dir, schema_docs, plain_schema_files)
|
|
357
|
+
end
|
|
358
|
+
[schema_docs.uniq.sort, plain_schema_files.uniq.sort]
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
def collect_attachment(ref, part_dir, schema_docs, plain_schema_files)
|
|
362
|
+
if ref.end_with?(".exp")
|
|
363
|
+
add_exp_attachment(ref, part_dir, schema_docs, plain_schema_files)
|
|
364
|
+
elsif ref.end_with?(".html") && (m = %r{schema_docs/([^/]+)/}.match(ref))
|
|
365
|
+
schema_docs << m[1]
|
|
366
|
+
end
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
# .exp filerefs resolve to their source through Plan's
|
|
370
|
+
# plain_schemas→schemas rule; the owned schema_docs dir is the
|
|
371
|
+
# resolved file's parent, and the recorded plain_schemas path is
|
|
372
|
+
# the same suffix under plain_schemas/. Filerefs that resolve
|
|
373
|
+
# nowhere fall back to the Python's plain string parse.
|
|
374
|
+
def add_exp_attachment(ref, part_dir, schema_docs, plain_schema_files)
|
|
375
|
+
source = Plan.resolve_exp_source(ref, part_dir, repo_root)
|
|
376
|
+
if source
|
|
377
|
+
rel = source.relative_path_from(repo_root).to_s
|
|
378
|
+
plain_rel = rel.sub(%r{\Aschemas/}, "plain_schemas/")
|
|
379
|
+
.delete_prefix("plain_schemas/")
|
|
380
|
+
plain_schema_files << plain_rel
|
|
381
|
+
schema_docs << Pathname.new(plain_rel).parent.basename.to_s
|
|
382
|
+
elsif (m = %r{plain_schemas/(.+\.exp)\z}.match(ref))
|
|
383
|
+
plain_schema_files << m[1]
|
|
384
|
+
schema_docs << Pathname.new(m[1]).parent.basename.to_s
|
|
385
|
+
end
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
# -- Step D: copy into the bundle ---------------------------------
|
|
389
|
+
|
|
390
|
+
def copy_artifacts(site, iso_dir, schema_docs)
|
|
391
|
+
bundle_dir = base / "parts" / @part
|
|
392
|
+
bundle_site = bundle_dir / "site"
|
|
393
|
+
bundle_logs = bundle_dir / "logs"
|
|
394
|
+
prior = bundle_dir / "bundle.json"
|
|
395
|
+
previous = prior.exist? ? File.binread(prior) : nil
|
|
396
|
+
|
|
397
|
+
FileUtils.rm_rf(bundle_dir)
|
|
398
|
+
FileUtils.mkdir_p([bundle_site, bundle_logs])
|
|
399
|
+
File.binwrite(bundle_dir / "bundle.json.previous", previous) if previous
|
|
400
|
+
|
|
401
|
+
FileUtils.cp_r(iso_dir, bundle_site / iso_dir.basename)
|
|
402
|
+
copy_owned_dirs(site / "schema_docs", bundle_site / "schema_docs",
|
|
403
|
+
schema_docs)
|
|
404
|
+
plain = site / "plain_schemas"
|
|
405
|
+
FileUtils.cp_r(plain, bundle_site / "plain_schemas") if plain.exist?
|
|
406
|
+
|
|
407
|
+
[worktree_site.parent / "srl-build-#{@part}.log",
|
|
408
|
+
worktree_site.parent / "collection-output.yaml"].each do |log|
|
|
409
|
+
FileUtils.cp(log, bundle_logs / log.basename) if log.exist?
|
|
410
|
+
end
|
|
411
|
+
bundle_dir
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
def copy_owned_dirs(src_root, dst_root, owned)
|
|
415
|
+
return unless src_root.exist?
|
|
416
|
+
|
|
417
|
+
FileUtils.mkdir_p(dst_root)
|
|
418
|
+
owned.each do |name|
|
|
419
|
+
src = src_root / name
|
|
420
|
+
FileUtils.cp_r(src, dst_root / name) if src.exist?
|
|
421
|
+
end
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
# -- Step E: bundle.json -------------------------------------------
|
|
425
|
+
|
|
426
|
+
# A faithful port: the record is one flat shape, assembled from
|
|
427
|
+
# the plan entry, the identity, and the scan results.
|
|
428
|
+
def write_bundle(bundle_dir, plan, entry, identity, iso_name, sections,
|
|
429
|
+
schema_docs, plain_schema_files, anchors, unresolved_xrefs)
|
|
430
|
+
bundle = {
|
|
431
|
+
"version" => 1,
|
|
432
|
+
"part_dir" => entry["part_dir"],
|
|
433
|
+
"iso_name" => iso_name,
|
|
434
|
+
"source_dir" => entry["source_dir"],
|
|
435
|
+
"build_timestamp" => Time.now.utc.iso8601,
|
|
436
|
+
"build_commit" => plan["build_commit"],
|
|
437
|
+
"doc_id" => identity["doc_id"],
|
|
438
|
+
"year" => identity["year"],
|
|
439
|
+
"doc_hash" => entry["doc_hash"],
|
|
440
|
+
"deps_hash" => entry["deps_hash"],
|
|
441
|
+
"input_files" => entry["input_files"],
|
|
442
|
+
"exp_files" => entry["exp_files"],
|
|
443
|
+
"sections" => sections,
|
|
444
|
+
"schema_docs" => schema_docs,
|
|
445
|
+
"plain_schemas" => plain_schema_files,
|
|
446
|
+
"anchors" => anchors,
|
|
447
|
+
"unresolved_xrefs" => unresolved_xrefs,
|
|
448
|
+
"errors" => { "build_warnings" => [] },
|
|
449
|
+
}
|
|
450
|
+
File.write(bundle_dir / "bundle.json", JSON.pretty_generate(bundle))
|
|
451
|
+
end
|
|
452
|
+
|
|
453
|
+
def note_iso_name_mismatch(entry, iso_name)
|
|
454
|
+
return if entry["iso_name"] == iso_name
|
|
455
|
+
|
|
456
|
+
Utils.log "NOTE: plan iso_name=#{entry['iso_name'].inspect} but build " \
|
|
457
|
+
"produced #{iso_name.inspect} (stage prefix differs — using " \
|
|
458
|
+
"rendered value)"
|
|
459
|
+
end
|
|
460
|
+
# rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/ParameterLists, Layout/LineLength
|
|
461
|
+
end
|
|
462
|
+
end
|
|
463
|
+
end
|
data/lib/suma/delivery/plan.rb
CHANGED
|
@@ -34,6 +34,78 @@ module Suma
|
|
|
34
34
|
plan
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
+
class << self
|
|
38
|
+
# Identity of one part as its collection.yml bibdata declares it:
|
|
39
|
+
# the docid string, the docid type (the prefix for per-doc
|
|
40
|
+
# directory names when the id carries none), and the copyright
|
|
41
|
+
# year (:copyright-year: in document.adoc wins, falling back to
|
|
42
|
+
# bibdata.copyright.from).
|
|
43
|
+
def identity(part_dir)
|
|
44
|
+
coll = YAML.safe_load_file(File.join(part_dir.to_s, "collection.yml"),
|
|
45
|
+
aliases: true)
|
|
46
|
+
bibdata = coll["bibdata"] || {}
|
|
47
|
+
{ "doc_id" => fetch_doc_id(bibdata, part_dir),
|
|
48
|
+
"docid_type" => docid_node(bibdata)["type"].to_s,
|
|
49
|
+
"year" => derive_year(part_dir, bibdata) }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def fetch_doc_id(bibdata, part_dir)
|
|
53
|
+
id = docid_node(bibdata)["id"]
|
|
54
|
+
id || Pathname.new(part_dir).basename.to_s.delete_prefix("iso-")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def derive_year(part_dir, bibdata)
|
|
58
|
+
adoc = File.join(part_dir.to_s, "document.adoc")
|
|
59
|
+
return copyright_from_bibdata(bibdata) unless File.exist?(adoc)
|
|
60
|
+
|
|
61
|
+
File.foreach(adoc) do |line|
|
|
62
|
+
if (m = line.match(/\A:copyright-year:\s*(\d+)/))
|
|
63
|
+
return m[1]
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
copyright_from_bibdata(bibdata)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def copyright_from_bibdata(bibdata)
|
|
70
|
+
from = bibdata.dig("copyright", "from")
|
|
71
|
+
from ? from.to_s : "0000"
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def docid_node(bibdata)
|
|
75
|
+
node = bibdata["docid"] || {}
|
|
76
|
+
node = node.first if node.is_a?(Array)
|
|
77
|
+
node.is_a?(Hash) ? node : {}
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# collection.yml filerefs point at plain_schemas/ (build-time
|
|
81
|
+
# output); the source lives at schemas/ under the same suffix.
|
|
82
|
+
def resolve_exp_source(fileref, part_dir, repo_root)
|
|
83
|
+
raw = (Pathname.new(part_dir) + fileref).expand_path
|
|
84
|
+
return raw if raw.exist? && under_repo?(raw, repo_root)
|
|
85
|
+
|
|
86
|
+
candidate = Pathname.new(raw.to_s.sub("/plain_schemas/", "/schemas/"))
|
|
87
|
+
candidate.exist? ? candidate : nil
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def under_repo?(path, repo_root)
|
|
91
|
+
base = Pathname.new(repo_root).expand_path
|
|
92
|
+
path.expand_path.to_s.start_with?("#{base}/")
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def walk_docrefs(node, acc = [])
|
|
96
|
+
return acc unless node.is_a?(Hash)
|
|
97
|
+
|
|
98
|
+
docref = node["docref"]
|
|
99
|
+
[docref].compact.flatten.each do |entry|
|
|
100
|
+
next unless entry.is_a?(Hash)
|
|
101
|
+
|
|
102
|
+
acc << entry
|
|
103
|
+
walk_docrefs(entry, acc)
|
|
104
|
+
end
|
|
105
|
+
acc
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
37
109
|
private
|
|
38
110
|
|
|
39
111
|
def collection_path
|
|
@@ -77,8 +149,8 @@ module Suma
|
|
|
77
149
|
coll = YAML.safe_load_file(File.join(part_dir, "collection.yml"),
|
|
78
150
|
aliases: true)
|
|
79
151
|
bibdata = coll["bibdata"] || {}
|
|
80
|
-
doc_id = fetch_doc_id(bibdata, part_dir)
|
|
81
|
-
year = derive_year(part_dir, bibdata)
|
|
152
|
+
doc_id = self.class.fetch_doc_id(bibdata, part_dir)
|
|
153
|
+
year = self.class.derive_year(part_dir, bibdata)
|
|
82
154
|
exp_files = exp_sources(part_dir, coll)
|
|
83
155
|
inputs = inventory(part_dir) + exp_files
|
|
84
156
|
|
|
@@ -106,27 +178,11 @@ module Suma
|
|
|
106
178
|
end
|
|
107
179
|
|
|
108
180
|
def fetch_doc_id(bibdata, part_dir)
|
|
109
|
-
|
|
110
|
-
node = node.first if node.is_a?(Array)
|
|
111
|
-
id = node && node["id"]
|
|
112
|
-
id || part_dir.basename.to_s.delete_prefix("iso-")
|
|
181
|
+
self.class.fetch_doc_id(bibdata, Pathname.new(part_dir))
|
|
113
182
|
end
|
|
114
183
|
|
|
115
184
|
def derive_year(part_dir, bibdata)
|
|
116
|
-
|
|
117
|
-
return copyright_from_bibdata(bibdata) unless File.exist?(adoc)
|
|
118
|
-
|
|
119
|
-
File.foreach(adoc) do |line|
|
|
120
|
-
if (m = line.match(/\A:copyright-year:\s*(\d+)/))
|
|
121
|
-
return m[1]
|
|
122
|
-
end
|
|
123
|
-
end
|
|
124
|
-
copyright_from_bibdata(bibdata)
|
|
125
|
-
end
|
|
126
|
-
|
|
127
|
-
def copyright_from_bibdata(bibdata)
|
|
128
|
-
from = bibdata.dig("copyright", "from")
|
|
129
|
-
from ? from.to_s : "0000"
|
|
185
|
+
self.class.derive_year(part_dir, bibdata)
|
|
130
186
|
end
|
|
131
187
|
|
|
132
188
|
def inventory(part_dir)
|
|
@@ -148,7 +204,7 @@ module Suma
|
|
|
148
204
|
end
|
|
149
205
|
|
|
150
206
|
def exp_sources(part_dir, coll)
|
|
151
|
-
entries = walk_docrefs(coll["manifest"] || {})
|
|
207
|
+
entries = self.class.walk_docrefs(coll["manifest"] || {})
|
|
152
208
|
entries.filter_map do |entry|
|
|
153
209
|
ref = entry["fileref"]
|
|
154
210
|
next unless ref.is_a?(String) && ref.end_with?(".exp")
|
|
@@ -157,29 +213,6 @@ module Suma
|
|
|
157
213
|
end
|
|
158
214
|
end
|
|
159
215
|
|
|
160
|
-
def walk_docrefs(node, acc = [])
|
|
161
|
-
return acc unless node.is_a?(Hash)
|
|
162
|
-
|
|
163
|
-
docref = node["docref"]
|
|
164
|
-
[docref].compact.flatten.each do |entry|
|
|
165
|
-
next unless entry.is_a?(Hash)
|
|
166
|
-
|
|
167
|
-
acc << entry
|
|
168
|
-
walk_docrefs(entry, acc)
|
|
169
|
-
end
|
|
170
|
-
acc
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
# collection.yml filerefs point at plain_schemas/ (build-time
|
|
174
|
-
# output); the source lives at schemas/ under the same suffix.
|
|
175
|
-
def resolve_exp_source(fileref, part_dir)
|
|
176
|
-
raw = (part_dir + fileref).expand_path
|
|
177
|
-
return raw if raw.exist? && under_repo?(raw)
|
|
178
|
-
|
|
179
|
-
candidate = Pathname.new(raw.to_s.sub("/plain_schemas/", "/schemas/"))
|
|
180
|
-
candidate.exist? ? candidate : nil
|
|
181
|
-
end
|
|
182
|
-
|
|
183
216
|
def sha256_files(paths)
|
|
184
217
|
h = Digest::SHA256.new
|
|
185
218
|
paths.each { |path| hash_one_file(h, path) }
|
|
@@ -199,11 +232,15 @@ module Suma
|
|
|
199
232
|
|
|
200
233
|
def rel_or_abs(path)
|
|
201
234
|
path = Pathname.new(path)
|
|
202
|
-
under_repo?(path
|
|
235
|
+
if self.class.under_repo?(path, repo_root)
|
|
236
|
+
path.relative_path_from(repo_root).to_s
|
|
237
|
+
else
|
|
238
|
+
path.to_s
|
|
239
|
+
end
|
|
203
240
|
end
|
|
204
241
|
|
|
205
|
-
def
|
|
206
|
-
|
|
242
|
+
def resolve_exp_source(fileref, part_dir)
|
|
243
|
+
self.class.resolve_exp_source(fileref, part_dir, repo_root)
|
|
207
244
|
end
|
|
208
245
|
|
|
209
246
|
def build_commit
|
data/lib/suma/delivery.rb
CHANGED
|
@@ -5,6 +5,7 @@ module Suma
|
|
|
5
5
|
# per-part sites and resolve their unresolved cross-document references.
|
|
6
6
|
module Delivery
|
|
7
7
|
autoload :FrameNav, "suma/delivery/frame_nav"
|
|
8
|
+
autoload :Pack, "suma/delivery/pack"
|
|
8
9
|
autoload :Plan, "suma/delivery/plan"
|
|
9
10
|
autoload :SiteVerifier, "suma/delivery/site_verifier"
|
|
10
11
|
autoload :Stitcher, "suma/delivery/stitcher"
|
data/lib/suma/term_extractor.rb
CHANGED
|
@@ -84,7 +84,10 @@ module Suma
|
|
|
84
84
|
concept.localization(lang)
|
|
85
85
|
end
|
|
86
86
|
|
|
87
|
-
|
|
87
|
+
# Key files by the stable identifier (schema.entity), not the UUID:
|
|
88
|
+
# unchanged concepts keep their filename across regenerations, and
|
|
89
|
+
# diffs of generated datasets stay readable (iso-10303#820).
|
|
90
|
+
filename = "#{concept.data.id}.yaml"
|
|
88
91
|
File.write(File.join(output_dir, filename), doc.to_yamls,
|
|
89
92
|
encoding: "utf-8")
|
|
90
93
|
end
|
data/lib/suma/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: suma
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.26
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-11-
|
|
11
|
+
date: 2026-11-05 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: expressir
|
|
@@ -229,6 +229,7 @@ files:
|
|
|
229
229
|
- lib/suma/corpus_graph.rb
|
|
230
230
|
- lib/suma/delivery.rb
|
|
231
231
|
- lib/suma/delivery/frame_nav.rb
|
|
232
|
+
- lib/suma/delivery/pack.rb
|
|
232
233
|
- lib/suma/delivery/plan.rb
|
|
233
234
|
- lib/suma/delivery/site_verifier.rb
|
|
234
235
|
- lib/suma/delivery/stitcher.rb
|