suma 0.5.23 → 0.5.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +21 -10
- data/lib/suma/cli/build.rb +6 -0
- data/lib/suma/cli/core.rb +4 -1
- data/lib/suma/cli/{srl.rb → delivery.rb} +67 -27
- data/lib/suma/cli.rb +1 -1
- data/lib/suma/{srl → delivery}/frame_nav.rb +1 -1
- data/lib/suma/delivery/plan.rb +234 -0
- data/lib/suma/{srl → delivery}/site_verifier.rb +1 -1
- data/lib/suma/{srl → delivery}/stitcher.rb +1 -1
- data/lib/suma/delivery.rb +12 -0
- data/lib/suma/processor.rb +4 -1
- data/lib/suma/staged_collection_builder.rb +3 -0
- data/lib/suma/version.rb +1 -1
- data/lib/suma.rb +1 -1
- metadata +7 -6
- data/lib/suma/srl.rb +0 -11
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 84aed10cf4b395df03b1e9c479f211c35620ea1b25c7fdc820b53445b82aba35
|
|
4
|
+
data.tar.gz: 66ebda5f4630fe796071be9109c2f9a89f472a22d9904dd62bdf2f58e91c9e8a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 26085e6362b8acd67ba3d80b6aef6f7bcc64ff81317702d01f34f1b359b2e9944a83b0dde81c0d933022953d93cfc8d581b77064fa117b94724bbc08ccc49d6d
|
|
7
|
+
data.tar.gz: 2a5a8d7b7b10ceaee01719e18c8ce4431470311e1642db68ef27d2811e4ed47f000cbe770d534120460675e3a14eb36be73c0081314c28dd2af31112a1a686d1
|
data/.rubocop_todo.yml
CHANGED
|
@@ -77,6 +77,7 @@ Layout/LineLength:
|
|
|
77
77
|
- 'spec/suma/register_manifest_generator_spec.rb'
|
|
78
78
|
- 'spec/suma/schema_category_spec.rb'
|
|
79
79
|
- 'spec/suma/schema_comparer_integration_spec.rb'
|
|
80
|
+
- 'spec/suma/delivery/plan_spec.rb'
|
|
80
81
|
- 'spec/suma/schema_discovery_spec.rb'
|
|
81
82
|
- 'spec/suma/schema_exporter_spec.rb'
|
|
82
83
|
- 'spec/suma/schema_index_spec.rb'
|
|
@@ -87,9 +88,10 @@ Layout/LineLength:
|
|
|
87
88
|
- 'spec/suma/term_extractor_spec.rb'
|
|
88
89
|
- 'spec/suma/urn_spec.rb'
|
|
89
90
|
- 'suma.gemspec'
|
|
90
|
-
- 'spec/suma/
|
|
91
|
-
- 'spec/suma/
|
|
92
|
-
- 'spec/suma/
|
|
91
|
+
- 'spec/suma/delivery/frame_nav_spec.rb'
|
|
92
|
+
- 'spec/suma/delivery/site_verifier_spec.rb'
|
|
93
|
+
- 'spec/suma/delivery/stitcher_spec.rb'
|
|
94
|
+
- 'spec/suma/delivery/plan_spec.rb'
|
|
93
95
|
|
|
94
96
|
# Offense count: 2
|
|
95
97
|
# Configuration parameters: IgnoreLiteralBranches, IgnoreConstantBranches, IgnoreDuplicateElseBranch.
|
|
@@ -147,7 +149,7 @@ Metrics/AbcSize:
|
|
|
147
149
|
- 'lib/suma/worker_pool.rb'
|
|
148
150
|
- 'spec/suma/cli/build_integration_spec.rb'
|
|
149
151
|
- 'spec/suma/reproducibility_spec.rb'
|
|
150
|
-
- spec/suma/
|
|
152
|
+
- spec/suma/delivery/stitcher_spec.rb
|
|
151
153
|
|
|
152
154
|
# Offense count: 2
|
|
153
155
|
# Configuration parameters: CountComments, Max, CountAsOne, AllowedMethods, AllowedPatterns, inherit_mode.
|
|
@@ -259,6 +261,7 @@ Performance/CollectionLiteralInLoop:
|
|
|
259
261
|
- 'spec/suma/cli/export_spec.rb'
|
|
260
262
|
- 'spec/suma/cli_spec.rb'
|
|
261
263
|
- 'spec/suma/schema_comparer_integration_spec.rb'
|
|
264
|
+
- 'spec/suma/delivery/plan_spec.rb'
|
|
262
265
|
|
|
263
266
|
# Offense count: 2
|
|
264
267
|
# Configuration parameters: Prefixes, AllowedPatterns.
|
|
@@ -312,6 +315,7 @@ RSpec/ExampleLength:
|
|
|
312
315
|
- 'spec/suma/schema_category_spec.rb'
|
|
313
316
|
- 'spec/suma/schema_collection_rebuild_targets_spec.rb'
|
|
314
317
|
- 'spec/suma/schema_comparer_integration_spec.rb'
|
|
318
|
+
- 'spec/suma/delivery/plan_spec.rb'
|
|
315
319
|
- 'spec/suma/schema_comparer_spec.rb'
|
|
316
320
|
- 'spec/suma/schema_compiler_spec.rb'
|
|
317
321
|
- 'spec/suma/schema_discovery_spec.rb'
|
|
@@ -330,9 +334,10 @@ RSpec/ExampleLength:
|
|
|
330
334
|
- 'spec/suma/urn_resolver_spec.rb'
|
|
331
335
|
- 'spec/suma/urn_spec.rb'
|
|
332
336
|
- 'spec/suma/worker_pool_spec.rb'
|
|
333
|
-
- 'spec/suma/
|
|
334
|
-
- 'spec/suma/
|
|
335
|
-
- 'spec/suma/
|
|
337
|
+
- 'spec/suma/delivery/frame_nav_spec.rb'
|
|
338
|
+
- 'spec/suma/delivery/site_verifier_spec.rb'
|
|
339
|
+
- 'spec/suma/delivery/stitcher_spec.rb'
|
|
340
|
+
- 'spec/suma/delivery/plan_spec.rb'
|
|
336
341
|
|
|
337
342
|
# Offense count: 4
|
|
338
343
|
# Configuration parameters: Max, AllowedIdentifiers, AllowedPatterns.
|
|
@@ -340,18 +345,21 @@ RSpec/IndexedLet:
|
|
|
340
345
|
Exclude:
|
|
341
346
|
- 'spec/suma/cli/export_spec.rb'
|
|
342
347
|
- 'spec/suma/schema_comparer_integration_spec.rb'
|
|
348
|
+
- 'spec/suma/delivery/plan_spec.rb'
|
|
343
349
|
|
|
344
350
|
# Offense count: 22
|
|
345
351
|
# Configuration parameters: AssignmentOnly.
|
|
346
352
|
RSpec/InstanceVariable:
|
|
347
353
|
Exclude:
|
|
348
354
|
- 'spec/suma/schema_comparer_integration_spec.rb'
|
|
355
|
+
- 'spec/suma/delivery/plan_spec.rb'
|
|
349
356
|
|
|
350
357
|
# Offense count: 1
|
|
351
358
|
# This cop supports safe autocorrection (--autocorrect).
|
|
352
359
|
RSpec/IteratedExpectation:
|
|
353
360
|
Exclude:
|
|
354
361
|
- 'spec/suma/schema_comparer_integration_spec.rb'
|
|
362
|
+
- 'spec/suma/delivery/plan_spec.rb'
|
|
355
363
|
|
|
356
364
|
# Offense count: 1
|
|
357
365
|
# Configuration parameters: .
|
|
@@ -401,6 +409,7 @@ RSpec/MultipleExpectations:
|
|
|
401
409
|
- 'spec/suma/schema_cache_spec.rb'
|
|
402
410
|
- 'spec/suma/schema_category_spec.rb'
|
|
403
411
|
- 'spec/suma/schema_comparer_integration_spec.rb'
|
|
412
|
+
- 'spec/suma/delivery/plan_spec.rb'
|
|
404
413
|
- 'spec/suma/schema_compiler_spec.rb'
|
|
405
414
|
- 'spec/suma/schema_discovery_spec.rb'
|
|
406
415
|
- 'spec/suma/schema_exporter_spec.rb'
|
|
@@ -417,9 +426,10 @@ RSpec/MultipleExpectations:
|
|
|
417
426
|
- 'spec/suma/term_extractor_spec.rb'
|
|
418
427
|
- 'spec/suma/urn_resolver_spec.rb'
|
|
419
428
|
- 'spec/suma/worker_pool_spec.rb'
|
|
420
|
-
- 'spec/suma/
|
|
421
|
-
- 'spec/suma/
|
|
422
|
-
- 'spec/suma/
|
|
429
|
+
- 'spec/suma/delivery/frame_nav_spec.rb'
|
|
430
|
+
- 'spec/suma/delivery/site_verifier_spec.rb'
|
|
431
|
+
- 'spec/suma/delivery/stitcher_spec.rb'
|
|
432
|
+
- 'spec/suma/delivery/plan_spec.rb'
|
|
423
433
|
|
|
424
434
|
# Offense count: 3
|
|
425
435
|
# Configuration parameters: Max, AllowedGroups.
|
|
@@ -445,6 +455,7 @@ RSpec/SpecFilePathFormat:
|
|
|
445
455
|
RSpec/StubbedMock:
|
|
446
456
|
Exclude:
|
|
447
457
|
- 'spec/suma/schema_comparer_integration_spec.rb'
|
|
458
|
+
- 'spec/suma/delivery/plan_spec.rb'
|
|
448
459
|
|
|
449
460
|
# Offense count: 2
|
|
450
461
|
# Configuration parameters: Max.
|
data/lib/suma/cli/build.rb
CHANGED
|
@@ -66,6 +66,11 @@ module Suma
|
|
|
66
66
|
option :no_valid, type: :boolean, default: false,
|
|
67
67
|
desc: "Skip Metanorma document validation " \
|
|
68
68
|
"(equivalent of metanorma --no-valid)"
|
|
69
|
+
option :unresolved_eref_markers, type: :boolean, default: false,
|
|
70
|
+
desc: "Emit <span class=\"unresolved-eref\"> " \
|
|
71
|
+
"markers for cross-document erefs " \
|
|
72
|
+
"(isodoc 3.7.9+; consumed by the " \
|
|
73
|
+
"SRL stitcher)"
|
|
69
74
|
|
|
70
75
|
def build(metanorma_site_manifest)
|
|
71
76
|
unless File.exist?(metanorma_site_manifest)
|
|
@@ -95,6 +100,7 @@ module Suma
|
|
|
95
100
|
sign_method: options[:sign],
|
|
96
101
|
sign_key: options[:sign_key],
|
|
97
102
|
no_valid: options[:no_valid],
|
|
103
|
+
unresolved_eref_markers: options[:unresolved_eref_markers],
|
|
98
104
|
**staged_options(options),
|
|
99
105
|
).run
|
|
100
106
|
end
|
data/lib/suma/cli/core.rb
CHANGED
|
@@ -58,6 +58,9 @@ module Suma
|
|
|
58
58
|
desc: "Signing key (sigstore signs keylessly when omitted)"
|
|
59
59
|
option :no_valid, type: :boolean, default: false,
|
|
60
60
|
desc: "Skip Metanorma document validation"
|
|
61
|
+
option :unresolved_eref_markers, type: :boolean, default: false,
|
|
62
|
+
desc: "Emit unresolved-eref span " \
|
|
63
|
+
"markers for the stitcher"
|
|
61
64
|
def build(_site_manifest)
|
|
62
65
|
Cli::Build.start
|
|
63
66
|
end
|
|
@@ -184,7 +187,7 @@ module Suma
|
|
|
184
187
|
subcommand "validate", Cli::Validate
|
|
185
188
|
|
|
186
189
|
desc "srl SUBCOMMAND ...ARGS", "SRL delivery: stitch per-part bundles into one site"
|
|
187
|
-
subcommand "
|
|
190
|
+
subcommand "delivery", Cli::Delivery
|
|
188
191
|
|
|
189
192
|
desc "resolve [URN]", "Resolve URNs to canonical JSON + HTML artifacts" # Mirrors so Core accepts the switches Cli::Resolve reparses; the
|
|
190
193
|
# inner class owns their declarations.
|
|
@@ -5,10 +5,13 @@ require "thor"
|
|
|
5
5
|
|
|
6
6
|
module Suma
|
|
7
7
|
module Cli
|
|
8
|
-
#
|
|
9
|
-
# rendered per-part bundles into one stitched
|
|
10
|
-
# cross-document references
|
|
11
|
-
|
|
8
|
+
# Stitched-delivery commands (introduced for TCSC410303-2952): enumerate
|
|
9
|
+
# parts, merge independently rendered per-part bundles into one stitched
|
|
10
|
+
# site, resolve their unresolved cross-document references, and verify
|
|
11
|
+
# the result. Artifact names (SRL/SMRL/...) are corpus-side; this
|
|
12
|
+
# surface is name-neutral and driven by whatever manifests the caller
|
|
13
|
+
# points it at.
|
|
14
|
+
class Delivery < Thor
|
|
12
15
|
desc "verify SITE_DIR",
|
|
13
16
|
"Verify every link in the stitched site resolves: cross-file " \
|
|
14
17
|
"hrefs to files, fragments to anchors in their target. The " \
|
|
@@ -20,34 +23,15 @@ module Suma
|
|
|
20
23
|
desc: "Skip fragment checking (file existence only)"
|
|
21
24
|
|
|
22
25
|
def verify(site_dir)
|
|
23
|
-
result = Suma::
|
|
26
|
+
result = Suma::Delivery::SiteVerifier.new(site_dir,
|
|
27
|
+
**verify_options).verify
|
|
24
28
|
log_verify(result)
|
|
25
29
|
raise "broken links found (#{result.broken_count})" unless
|
|
26
30
|
result.broken_count.zero?
|
|
27
31
|
end
|
|
28
32
|
|
|
29
|
-
private
|
|
30
|
-
|
|
31
|
-
def verify_options
|
|
32
|
-
{ check_fragments: !options[:no_fragments] }
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def log_verify(result)
|
|
36
|
-
report = result.report
|
|
37
|
-
write_report(report) if options[:out]
|
|
38
|
-
Utils.log "Links: #{report['total_links']} total, " \
|
|
39
|
-
"#{report['broken_links'].size} broken; " \
|
|
40
|
-
"fragments: #{report['total_fragments']} total, " \
|
|
41
|
-
"#{report['broken_fragments'].size} broken; " \
|
|
42
|
-
"#{report['cross_document_links']} cross-document"
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
def write_report(report)
|
|
46
|
-
File.write(options[:out], JSON.pretty_generate(report))
|
|
47
|
-
end
|
|
48
|
-
|
|
49
33
|
desc "stitch BUNDLES_DIR",
|
|
50
|
-
"Merge per-part
|
|
34
|
+
"Merge per-part bundles (each bundle.json + site/) into one " \
|
|
51
35
|
"_site/, resolving every unresolved-eref span through the " \
|
|
52
36
|
"anchor/doc/attachment registries; writes the audit records " \
|
|
53
37
|
"and the merged index"
|
|
@@ -68,11 +52,59 @@ module Suma
|
|
|
68
52
|
desc: "Branch recorded in build-info.json"
|
|
69
53
|
|
|
70
54
|
def stitch(bundles_dir)
|
|
71
|
-
result = Suma::
|
|
55
|
+
result = Suma::Delivery::Stitcher.new(**stitch_options(bundles_dir)).stitch
|
|
72
56
|
Utils.log "Stitched #{result.parts.size} into #{options[:out]}: " \
|
|
73
57
|
"#{result.resolved} ok, #{result.broken} broken"
|
|
74
58
|
end
|
|
75
59
|
|
|
60
|
+
desc "plan", "Enumerate parts from the collection manifest and " \
|
|
61
|
+
"write plan.json for the stitcher: per-part identity " \
|
|
62
|
+
"(iso_name, doc_id, year), input inventory, and " \
|
|
63
|
+
"doc/deps content hashes"
|
|
64
|
+
option :collection, type: :string, default: "collection-resources.yml",
|
|
65
|
+
desc: "Collection manifest listing one docref per " \
|
|
66
|
+
"part, relative to --repo-root"
|
|
67
|
+
option :repo_root, type: :string, default: ".",
|
|
68
|
+
desc: "Repository root"
|
|
69
|
+
option :out, type: :string, required: true,
|
|
70
|
+
desc: "Output plan.json path"
|
|
71
|
+
|
|
72
|
+
def plan
|
|
73
|
+
result = Suma::Delivery::Plan.new(**plan_options).generate
|
|
74
|
+
Utils.log "Planned #{result['parts_count']} parts " \
|
|
75
|
+
"from #{result['source_collection']}"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
desc "frame-nav SITE_DIR", "Post-process a stitched site for " \
|
|
79
|
+
"frame navigation: strip the index chrome, " \
|
|
80
|
+
"build the top-level index, wire onclick " \
|
|
81
|
+
"index links, inject the frameset script, " \
|
|
82
|
+
"and add target attributes"
|
|
83
|
+
def frame_nav(site_dir)
|
|
84
|
+
Suma::Delivery::FrameNav.new(site_dir).process
|
|
85
|
+
Utils.log "Frame navigation applied to #{site_dir}"
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
private
|
|
89
|
+
|
|
90
|
+
def verify_options
|
|
91
|
+
{ check_fragments: !options[:no_fragments] }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def log_verify(result)
|
|
95
|
+
report = result.report
|
|
96
|
+
write_report(report) if options[:out]
|
|
97
|
+
Utils.log "Links: #{report['total_links']} total, " \
|
|
98
|
+
"#{report['broken_links'].size} broken; " \
|
|
99
|
+
"fragments: #{report['total_fragments']} total, " \
|
|
100
|
+
"#{report['broken_fragments'].size} broken; " \
|
|
101
|
+
"#{report['cross_document_links']} cross-document"
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def write_report(report)
|
|
105
|
+
File.write(options[:out], JSON.pretty_generate(report))
|
|
106
|
+
end
|
|
107
|
+
|
|
76
108
|
def stitch_options(bundles_dir)
|
|
77
109
|
{
|
|
78
110
|
bundles_dir: bundles_dir,
|
|
@@ -84,6 +116,14 @@ module Suma
|
|
|
84
116
|
source_branch: options[:source_branch],
|
|
85
117
|
}
|
|
86
118
|
end
|
|
119
|
+
|
|
120
|
+
def plan_options
|
|
121
|
+
{
|
|
122
|
+
repo_root: options[:repo_root],
|
|
123
|
+
collection: options[:collection],
|
|
124
|
+
out: options[:out],
|
|
125
|
+
}
|
|
126
|
+
end
|
|
87
127
|
end
|
|
88
128
|
end
|
|
89
129
|
end
|
data/lib/suma/cli.rb
CHANGED
|
@@ -12,7 +12,7 @@ module Suma
|
|
|
12
12
|
autoload :ExtractTerms, "suma/cli/extract_terms"
|
|
13
13
|
autoload :GenerateRegister, "suma/cli/generate_register"
|
|
14
14
|
autoload :GenerateSchemas, "suma/cli/generate_schemas"
|
|
15
|
-
autoload :
|
|
15
|
+
autoload :Delivery, "suma/cli/delivery"
|
|
16
16
|
autoload :IngestTranslations, "suma/cli/ingest_translations"
|
|
17
17
|
autoload :PackageGcr, "suma/cli/package_gcr"
|
|
18
18
|
autoload :Reformat, "suma/cli/reformat"
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "json"
|
|
5
|
+
require "open3"
|
|
6
|
+
require "yaml"
|
|
7
|
+
|
|
8
|
+
module Suma
|
|
9
|
+
module Delivery
|
|
10
|
+
# Enumerates the parts of a stitched delivery from the collection
|
|
11
|
+
# manifest and writes plan.json: one entry per docref with the part's
|
|
12
|
+
# identity, input inventory, and content hashes. Drives the
|
|
13
|
+
# per-document stitcher pipeline (TCSC410303-2952 Stage A).
|
|
14
|
+
class Plan
|
|
15
|
+
DEFAULT_COLLECTION = "collection-resources.yml"
|
|
16
|
+
|
|
17
|
+
attr_reader :repo_root, :collection, :out_path
|
|
18
|
+
|
|
19
|
+
def initialize(repo_root:, out:, collection: DEFAULT_COLLECTION)
|
|
20
|
+
@repo_root = Pathname.new(File.expand_path(repo_root))
|
|
21
|
+
@collection = collection
|
|
22
|
+
@out_path = Pathname.new(out)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def generate
|
|
26
|
+
docrefs = manifest_docrefs(load_collection_manifest)
|
|
27
|
+
unless docrefs.any?
|
|
28
|
+
raise "No manifest.docref entries in #{collection_path}"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
parts = plan_parts(docrefs)
|
|
32
|
+
plan = build_plan(parts)
|
|
33
|
+
write_plan(plan)
|
|
34
|
+
plan
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def collection_path
|
|
40
|
+
repo_root + collection
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def load_collection_manifest
|
|
44
|
+
YAML.safe_load_file(collection_path, aliases: true)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def manifest_docrefs(manifest)
|
|
48
|
+
node = manifest.dig("manifest", "docref")
|
|
49
|
+
[node].compact.flatten
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def plan_parts(docrefs)
|
|
53
|
+
docrefs.filter_map do |ref|
|
|
54
|
+
next unless ref.is_a?(Hash) && ref["file"]
|
|
55
|
+
|
|
56
|
+
part_collection = repo_root + ref["file"]
|
|
57
|
+
unless part_collection.exist?
|
|
58
|
+
Utils.log "WARN: docref not found: #{part_collection}"
|
|
59
|
+
next
|
|
60
|
+
end
|
|
61
|
+
analyze_part(part_collection.parent)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def build_plan(parts)
|
|
66
|
+
{
|
|
67
|
+
"version" => 1,
|
|
68
|
+
"source_collection" => collection,
|
|
69
|
+
"build_commit" => build_commit,
|
|
70
|
+
"build_timestamp" => Time.now.utc.iso8601,
|
|
71
|
+
"parts_count" => parts.size,
|
|
72
|
+
"parts" => parts,
|
|
73
|
+
}
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def analyze_part(part_dir)
|
|
77
|
+
coll = YAML.safe_load_file(File.join(part_dir, "collection.yml"),
|
|
78
|
+
aliases: true)
|
|
79
|
+
bibdata = coll["bibdata"] || {}
|
|
80
|
+
doc_id = fetch_doc_id(bibdata, part_dir)
|
|
81
|
+
year = derive_year(part_dir, bibdata)
|
|
82
|
+
exp_files = exp_sources(part_dir, coll)
|
|
83
|
+
inputs = inventory(part_dir) + exp_files
|
|
84
|
+
|
|
85
|
+
part_identity(part_dir, doc_id,
|
|
86
|
+
year).merge(part_hashes(inputs, exp_files))
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def part_identity(part_dir, doc_id, year)
|
|
90
|
+
{
|
|
91
|
+
"part_dir" => part_dir.basename.to_s,
|
|
92
|
+
"source_dir" => part_dir.relative_path_from(repo_root).to_s,
|
|
93
|
+
"iso_name" => "ISO_#{doc_id}_#{year}",
|
|
94
|
+
"doc_id" => doc_id,
|
|
95
|
+
"year" => year,
|
|
96
|
+
}
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def part_hashes(inputs, exp_files)
|
|
100
|
+
{
|
|
101
|
+
"doc_hash" => "sha256:#{sha256_files(inputs)}",
|
|
102
|
+
"deps_hash" => "sha256:#{sha256_files(exp_files)}",
|
|
103
|
+
"input_files" => inputs.map { |p| rel_or_abs(p) },
|
|
104
|
+
"exp_files" => exp_files.map { |p| rel_or_abs(p) },
|
|
105
|
+
}
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def fetch_doc_id(bibdata, part_dir)
|
|
109
|
+
node = bibdata["docid"] || {}
|
|
110
|
+
node = node.first if node.is_a?(Array)
|
|
111
|
+
id = node && node["id"]
|
|
112
|
+
id || part_dir.basename.to_s.delete_prefix("iso-")
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def derive_year(part_dir, bibdata)
|
|
116
|
+
adoc = File.join(part_dir, "document.adoc")
|
|
117
|
+
return copyright_from_bibdata(bibdata) unless File.exist?(adoc)
|
|
118
|
+
|
|
119
|
+
File.foreach(adoc) do |line|
|
|
120
|
+
if (m = line.match(/\A:copyright-year:\s*(\d+)/))
|
|
121
|
+
return m[1]
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
copyright_from_bibdata(bibdata)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def copyright_from_bibdata(bibdata)
|
|
128
|
+
from = bibdata.dig("copyright", "from")
|
|
129
|
+
from ? from.to_s : "0000"
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def inventory(part_dir)
|
|
133
|
+
inputs = [File.join(part_dir, "collection.yml")]
|
|
134
|
+
adoc = File.join(part_dir, "document.adoc")
|
|
135
|
+
inputs << adoc if File.exist?(adoc)
|
|
136
|
+
inputs += section_files(part_dir)
|
|
137
|
+
cover = File.join(part_dir, "cover.html")
|
|
138
|
+
inputs << cover if File.exist?(cover)
|
|
139
|
+
inputs
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def section_files(part_dir)
|
|
143
|
+
sections = File.join(part_dir, "sections")
|
|
144
|
+
return [] unless File.directory?(sections)
|
|
145
|
+
|
|
146
|
+
Dir.glob(File.join(sections, "**/*")).select { |p| File.file?(p) }
|
|
147
|
+
.sort.map { |p| Pathname.new(p) }
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def exp_sources(part_dir, coll)
|
|
151
|
+
entries = walk_docrefs(coll["manifest"] || {})
|
|
152
|
+
entries.filter_map do |entry|
|
|
153
|
+
ref = entry["fileref"]
|
|
154
|
+
next unless ref.is_a?(String) && ref.end_with?(".exp")
|
|
155
|
+
|
|
156
|
+
resolve_exp_source(ref, part_dir)
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def walk_docrefs(node, acc = [])
|
|
161
|
+
return acc unless node.is_a?(Hash)
|
|
162
|
+
|
|
163
|
+
docref = node["docref"]
|
|
164
|
+
[docref].compact.flatten.each do |entry|
|
|
165
|
+
next unless entry.is_a?(Hash)
|
|
166
|
+
|
|
167
|
+
acc << entry
|
|
168
|
+
walk_docrefs(entry, acc)
|
|
169
|
+
end
|
|
170
|
+
acc
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# collection.yml filerefs point at plain_schemas/ (build-time
|
|
174
|
+
# output); the source lives at schemas/ under the same suffix.
|
|
175
|
+
def resolve_exp_source(fileref, part_dir)
|
|
176
|
+
raw = (part_dir + fileref).expand_path
|
|
177
|
+
return raw if raw.exist? && under_repo?(raw)
|
|
178
|
+
|
|
179
|
+
candidate = Pathname.new(raw.to_s.sub("/plain_schemas/", "/schemas/"))
|
|
180
|
+
candidate.exist? ? candidate : nil
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def sha256_files(paths)
|
|
184
|
+
h = Digest::SHA256.new
|
|
185
|
+
paths.each { |path| hash_one_file(h, path) }
|
|
186
|
+
h.hexdigest
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def hash_one_file(hash, path)
|
|
190
|
+
hash.update(rel_or_abs(path))
|
|
191
|
+
hash.update("\0")
|
|
192
|
+
File.open(path, "rb") do |f|
|
|
193
|
+
while (chunk = f.read(65_536))
|
|
194
|
+
hash.update(chunk)
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
hash.update("\0\0")
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def rel_or_abs(path)
|
|
201
|
+
path = Pathname.new(path)
|
|
202
|
+
under_repo?(path) ? path.relative_path_from(repo_root).to_s : path.to_s
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def under_repo?(path)
|
|
206
|
+
path.expand_path.to_s.start_with?("#{repo_root.expand_path}/")
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def build_commit
|
|
210
|
+
@build_commit ||=
|
|
211
|
+
begin
|
|
212
|
+
git("rev-parse", "HEAD")
|
|
213
|
+
rescue StandardError
|
|
214
|
+
"unknown"
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def git(*args)
|
|
219
|
+
out, err, status = Open3.capture3("git", "-C", repo_root.to_s, *args)
|
|
220
|
+
raise "git #{args.first} failed: #{err}" unless status.success?
|
|
221
|
+
|
|
222
|
+
out.strip
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def write_plan(plan)
|
|
226
|
+
out_path.parent.mkpath
|
|
227
|
+
File.write(out_path, JSON.pretty_generate(plan))
|
|
228
|
+
Utils.log "plan.json written: #{out_path} " \
|
|
229
|
+
"(parts: #{plan['parts_count']}, " \
|
|
230
|
+
"commit: #{plan['build_commit']})"
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
end
|
|
@@ -5,7 +5,7 @@ require "pathname"
|
|
|
5
5
|
require "leptris"
|
|
6
6
|
|
|
7
7
|
module Suma
|
|
8
|
-
module
|
|
8
|
+
module Delivery
|
|
9
9
|
# Site-wide link verification (port of site_manifest.py verify_links):
|
|
10
10
|
# every cross-file href in the stitched site must resolve to a file,
|
|
11
11
|
# and every fragment must resolve to an anchor in its target file.
|
|
@@ -7,7 +7,7 @@ require "pathname"
|
|
|
7
7
|
require "leptris"
|
|
8
8
|
|
|
9
9
|
module Suma
|
|
10
|
-
module
|
|
10
|
+
module Delivery
|
|
11
11
|
# Merge independently rendered per-part sites into a single _site/,
|
|
12
12
|
# rewriting each part's <span class="unresolved-eref"
|
|
13
13
|
# data-bibitemid="…"> markers into cross-document <a href> elements
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Suma
|
|
4
|
+
# SRL delivery subsystem (TCSC410303-2952): merge independently rendered
|
|
5
|
+
# per-part sites and resolve their unresolved cross-document references.
|
|
6
|
+
module Delivery
|
|
7
|
+
autoload :FrameNav, "suma/delivery/frame_nav"
|
|
8
|
+
autoload :Plan, "suma/delivery/plan"
|
|
9
|
+
autoload :SiteVerifier, "suma/delivery/site_verifier"
|
|
10
|
+
autoload :Stitcher, "suma/delivery/stitcher"
|
|
11
|
+
end
|
|
12
|
+
end
|
data/lib/suma/processor.rb
CHANGED
|
@@ -18,7 +18,7 @@ module Suma
|
|
|
18
18
|
staged_reinflate_shard: nil,
|
|
19
19
|
incremental: false, state_file: nil, jobs: 1,
|
|
20
20
|
provenance_file: nil, sign_method: nil, sign_key: nil,
|
|
21
|
-
no_valid: false)
|
|
21
|
+
no_valid: false, unresolved_eref_markers: false)
|
|
22
22
|
@metanorma_yaml_path = metanorma_yaml_path
|
|
23
23
|
@schemas_all_path = schemas_all_path
|
|
24
24
|
@compile_flag = compile
|
|
@@ -40,6 +40,7 @@ module Suma
|
|
|
40
40
|
# Skip Metanorma document validation (metanorma `--no-valid`). Opt-in:
|
|
41
41
|
# for collections whose flavor's validators are still in flight.
|
|
42
42
|
@no_valid = no_valid
|
|
43
|
+
@unresolved_eref_markers = unresolved_eref_markers
|
|
43
44
|
end
|
|
44
45
|
|
|
45
46
|
def run
|
|
@@ -189,6 +190,7 @@ module Suma
|
|
|
189
190
|
output_directory: output_directory,
|
|
190
191
|
coverpage: collection_coverpage(collection_config),
|
|
191
192
|
no_valid: @no_valid,
|
|
193
|
+
unresolved_eref_markers: @unresolved_eref_markers,
|
|
192
194
|
workers: @staged_workers,
|
|
193
195
|
batch_size: @staged_batch_size ||
|
|
194
196
|
StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
|
|
@@ -213,6 +215,7 @@ module Suma
|
|
|
213
215
|
def compile_flags
|
|
214
216
|
{ install_fonts: false }.tap do |flags|
|
|
215
217
|
flags[:novalid] = true if @no_valid
|
|
218
|
+
flags[:unresolved_eref_markers] = true if @unresolved_eref_markers
|
|
216
219
|
end
|
|
217
220
|
end
|
|
218
221
|
|
|
@@ -54,6 +54,7 @@ module Suma
|
|
|
54
54
|
# (a data-declaring initializer)
|
|
55
55
|
def initialize(collection_config_path:, output_directory:, coverpage: nil,
|
|
56
56
|
store_dir: nil, formats: %i[xml html], no_valid: false,
|
|
57
|
+
unresolved_eref_markers: false,
|
|
57
58
|
workers: 1, batch_size: DEFAULT_BATCH_SIZE,
|
|
58
59
|
shard: nil, isolate: nil, reinflate_shard: nil) # rubocop:disable Metrics/ParameterLists
|
|
59
60
|
@config_path = collection_config_path
|
|
@@ -63,6 +64,7 @@ module Suma
|
|
|
63
64
|
@store_dir = store_dir || File.join(output_directory, STORE_DIRNAME)
|
|
64
65
|
@formats = formats
|
|
65
66
|
@no_valid = no_valid
|
|
67
|
+
@unresolved_eref_markers = unresolved_eref_markers
|
|
66
68
|
@workers = [workers.to_i, 1].max
|
|
67
69
|
@batch_size = [batch_size.to_i, 1].max
|
|
68
70
|
@shard = shard
|
|
@@ -451,6 +453,7 @@ module Suma
|
|
|
451
453
|
def compile_flags
|
|
452
454
|
{ install_fonts: false }.tap do |flags|
|
|
453
455
|
flags[:novalid] = true if @no_valid
|
|
456
|
+
flags[:unresolved_eref_markers] = true if @unresolved_eref_markers
|
|
454
457
|
end
|
|
455
458
|
end
|
|
456
459
|
|
data/lib/suma/version.rb
CHANGED
data/lib/suma.rb
CHANGED
|
@@ -34,7 +34,7 @@ module Suma
|
|
|
34
34
|
autoload :NullCache, "suma/null_cache"
|
|
35
35
|
autoload :OutputCanonicalizer, "suma/output_canonicalizer"
|
|
36
36
|
autoload :OwlProjection, "suma/owl_projection"
|
|
37
|
-
autoload :
|
|
37
|
+
autoload :Delivery, "suma/delivery"
|
|
38
38
|
autoload :RegisterManifestGenerator, "suma/register_manifest_generator"
|
|
39
39
|
autoload :SchemaCache, "suma/schema_cache"
|
|
40
40
|
autoload :SchemaCategory, "suma/schema_category"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: suma
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.24
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
@@ -212,6 +212,7 @@ files:
|
|
|
212
212
|
- lib/suma/cli/compare.rb
|
|
213
213
|
- lib/suma/cli/convert_jsdai.rb
|
|
214
214
|
- lib/suma/cli/core.rb
|
|
215
|
+
- lib/suma/cli/delivery.rb
|
|
215
216
|
- lib/suma/cli/export.rb
|
|
216
217
|
- lib/suma/cli/extract_terms.rb
|
|
217
218
|
- lib/suma/cli/generate_register.rb
|
|
@@ -220,13 +221,17 @@ files:
|
|
|
220
221
|
- lib/suma/cli/package_gcr.rb
|
|
221
222
|
- lib/suma/cli/reformat.rb
|
|
222
223
|
- lib/suma/cli/resolve.rb
|
|
223
|
-
- lib/suma/cli/srl.rb
|
|
224
224
|
- lib/suma/cli/validate.rb
|
|
225
225
|
- lib/suma/cli/validate_links.rb
|
|
226
226
|
- lib/suma/closure_validator.rb
|
|
227
227
|
- lib/suma/collection_config.rb
|
|
228
228
|
- lib/suma/collection_manifest.rb
|
|
229
229
|
- lib/suma/corpus_graph.rb
|
|
230
|
+
- lib/suma/delivery.rb
|
|
231
|
+
- lib/suma/delivery/frame_nav.rb
|
|
232
|
+
- lib/suma/delivery/plan.rb
|
|
233
|
+
- lib/suma/delivery/site_verifier.rb
|
|
234
|
+
- lib/suma/delivery/stitcher.rb
|
|
230
235
|
- lib/suma/eengine.rb
|
|
231
236
|
- lib/suma/eengine/errors.rb
|
|
232
237
|
- lib/suma/eengine/wrapper.rb
|
|
@@ -269,10 +274,6 @@ files:
|
|
|
269
274
|
- lib/suma/schema_template/plain.rb
|
|
270
275
|
- lib/suma/site_build.rb
|
|
271
276
|
- lib/suma/site_config.rb
|
|
272
|
-
- lib/suma/srl.rb
|
|
273
|
-
- lib/suma/srl/frame_nav.rb
|
|
274
|
-
- lib/suma/srl/site_verifier.rb
|
|
275
|
-
- lib/suma/srl/stitcher.rb
|
|
276
277
|
- lib/suma/staged_collection_builder.rb
|
|
277
278
|
- lib/suma/svg_quality.rb
|
|
278
279
|
- lib/suma/svg_quality/batch_report.rb
|
data/lib/suma/srl.rb
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Suma
|
|
4
|
-
# SRL delivery subsystem (TCSC410303-2952): merge independently rendered
|
|
5
|
-
# per-part sites and resolve their unresolved cross-document references.
|
|
6
|
-
module Srl
|
|
7
|
-
autoload :FrameNav, "suma/srl/frame_nav"
|
|
8
|
-
autoload :SiteVerifier, "suma/srl/site_verifier"
|
|
9
|
-
autoload :Stitcher, "suma/srl/stitcher"
|
|
10
|
-
end
|
|
11
|
-
end
|