metanorma 1.4.3 → 1.4.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d977a50a2b1b4c0a016439be744d32ef51fad6a4f0e5869f0ea31654065bb584
4
- data.tar.gz: 3da646653cdea95e20f7e0c5ee07a9ccf301b4f73506d1f65059e03b71c064de
3
+ metadata.gz: b300675f26c29369f59151fc665c176cd639b324ae7b3f9f9a25d2202740d45c
4
+ data.tar.gz: b1ddfe077b756b1d53093c1304099fbab5df7808220e0ce20c2b5d025dc1e011
5
5
  SHA512:
6
- metadata.gz: be99d0c2c11598c9e5dcd05df5da7a1bcdf49a161a56928ef8c1873bae349b401823d5f112b2aa4e0350bc20511125921c626fbd4c7ef862f6291476a409281c
7
- data.tar.gz: aaba4884d254314d5f3436d6f709503a34810398f093e1010cb4186c2bb1fcd7ebc1b41725dc0542db3ce4a7896f151b826a146332d3a4a2fffb0be47f9a3482
6
+ metadata.gz: 988cf0f24e900c751533ecf973ba0cc73b686640bdaf5933e0078a0cfd45c6f5165f4fe7ddf438519cec42b2082e73f0b8c0d1905ff4a28d310e952ab8e68b13
7
+ data.tar.gz: af28119205eac475051ff399537f9b4d7a4d9f2d7c5954cac11f1360c1d0d612d3faf2a76283edac0088cf717ed0c5dfa9a0bec9da66d330375f5c9170950514
@@ -9,35 +9,7 @@ on:
9
9
  pull_request:
10
10
 
11
11
  jobs:
12
- rake:
13
- name: Test on Ruby ${{ matrix.ruby }} ${{ matrix.os }}
14
- runs-on: ${{ matrix.os }}
15
- continue-on-error: ${{ matrix.experimental }}
16
- strategy:
17
- fail-fast: false
18
- matrix:
19
- ruby: [ '3.0', '2.7', '2.6', '2.5' ]
20
- os: [ ubuntu-latest, windows-latest, macos-latest ]
21
- experimental: [ false ]
22
- steps:
23
- - uses: actions/checkout@v2
24
- with:
25
- submodules: true
26
-
27
- - uses: ruby/setup-ruby@v1
28
- with:
29
- ruby-version: ${{ matrix.ruby }}
30
- bundler-cache: true
31
-
32
- - run: bundle exec rake
33
-
34
- tests-passed:
35
- needs: rake
36
- runs-on: ubuntu-latest
37
- steps:
38
- - uses: peter-evans/repository-dispatch@v1
39
- with:
40
- token: ${{ secrets.METANORMA_CI_PAT_TOKEN || secrets.GITHUB_TOKEN }}
41
- repository: ${{ github.repository }}
42
- event-type: tests-passed
43
- client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'
12
+ notify:
13
+ uses: metanorma/metanorma-build-scripts/.github/workflows/inkscape-rake.yml@main
14
+ secrets:
15
+ pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}
data/README.adoc CHANGED
@@ -247,6 +247,13 @@ The options hash has the same structure it does when invoked in metanorma-cli:
247
247
  `:relaton`: exports the bibdata Relaton XML description of the document (which is part of its Metanorma XML)
248
248
  to the nominated directory
249
249
 
250
+ == Threaded execution
251
+
252
+ Metanorma has threaded execution, to generate output documents from the same Presentation XML input more quickly.
253
+ Similar to https://github.com/relaton/relaton[relaton], the `METANORMA_PARALLEL` environment variable
254
+ can be used to override the default number of parallel fetches used.
255
+
256
+
250
257
  == Origin of name
251
258
 
252
259
  *Meta-* is a prefix of Greek origin ("μετα") for "`with`" "`after`".
@@ -16,7 +16,7 @@ module Metanorma
16
16
  # @param options [Hash]
17
17
  # @option options [String] :coverpage cover page HTML (Liquid template)
18
18
  # @option options [Array<Symbol>] :format list of formats (xml,html,doc,pdf)
19
- # @option options [String] :ourput_folder output directory
19
+ # @option options [String] :output_folder output directory
20
20
  #
21
21
  # We presuppose that the bibdata of the document is equivalent to that of
22
22
  # the collection, and that the flavour gem can sensibly process it. We may
@@ -52,7 +52,13 @@ module Metanorma
52
52
  end
53
53
 
54
54
  def dir_name_cleanse(name)
55
- name.gsub(/[<>:"|?*]/, "_")
55
+ path = Pathname.new(name)
56
+ clean_regex = /[<>:"|?*]/
57
+ fallback_sym = "_"
58
+ return name.gsub(clean_regex, fallback_sym) unless path.absolute?
59
+
60
+ File.join(path.dirname,
61
+ path.basename.to_s.gsub(clean_regex, fallback_sym))
56
62
  end
57
63
 
58
64
  # @param col [Metanorma::Collection] XML collection
@@ -9,6 +9,7 @@ require_relative "fontist_utils"
9
9
  require_relative "util"
10
10
  require_relative "sectionsplit"
11
11
  require_relative "extract"
12
+ require_relative "worker_pool"
12
13
 
13
14
  module Metanorma
14
15
  class Compile
@@ -33,7 +34,7 @@ module Metanorma
33
34
  extract(isodoc, options[:extract], options[:extract_type])
34
35
  FontistUtils.install_fonts(@processor, options) unless @fontist_installed
35
36
  @fontist_installed = true
36
- process_extensions(filename, extensions, file, isodoc, options)
37
+ process_exts(filename, extensions, file, isodoc, options)
37
38
  end
38
39
 
39
40
  def require_libraries(options)
@@ -119,6 +120,16 @@ module Metanorma
119
120
  File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
120
121
  end
121
122
 
123
+ def relaton_export(isodoc, options)
124
+ return unless options[:relaton]
125
+
126
+ xml = Nokogiri::XML(isodoc) { |config| config.huge }
127
+ bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
128
+ # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
129
+ # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
130
+ File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
131
+ end
132
+
122
133
  def export_output(fname, content, **options)
123
134
  mode = options[:binary] ? "wb" : "w:UTF-8"
124
135
  File.open(fname, mode) { |f| f.write content }
@@ -134,41 +145,68 @@ module Metanorma
134
145
  end
135
146
 
136
147
  # isodoc is Raw Metanorma XML
137
- def process_extensions(filename, extensions, file, isodoc, options)
138
- f = change_output_dir options
139
- name = { xml: f.sub(/\.[^.]+$/, ".xml"),
140
- presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
148
+ def process_exts(filename, extensions, file, isodoc, options)
149
+ f = File.expand_path(change_output_dir(options))
150
+ fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
151
+ orig_filename: File.expand_path(filename),
152
+ presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
153
+ @queue = ::Metanorma::WorkersPool
154
+ .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
141
155
  Util.sort_extensions_execution(extensions).each do |ext|
142
- file_extension = @processor.output_formats[ext]
143
- name[:out] = f.sub(/\.[^.]+$/, ".#{file_extension}")
144
- isodoc_options = get_isodoc_options(file, options, ext)
145
- if ext == :rxl
146
- relaton_export(isodoc, options.merge(relaton: name[:out]))
147
- elsif options[:passthrough_presentation_xml] && ext == :presentation
148
- FileUtils.cp filename, name[:presentationxml]
149
- elsif ext == :html && options[:sectionsplit]
150
- sectionsplit_convert(name[:xml], isodoc, name[:out], isodoc_options)
151
- else
152
- if ext == :pdf && FontistUtils.has_fonts_manifest?(@processor,
153
- options)
154
- isodoc_options[:mn2pdf] = {
155
- font_manifest: FontistUtils.location_manifest(@processor),
156
- }
157
- end
158
- process_extensions1(ext, name, isodoc, isodoc_options)
159
- end
160
- wrap_html(options, file_extension, name[:out])
156
+ process_ext(ext, file, isodoc, fnames, options)
157
+ end
158
+ @queue.shutdown
159
+ end
160
+
161
+ def process_ext(ext, file, isodoc, fnames, options)
162
+ fnames[:ext] = @processor.output_formats[ext]
163
+ fnames[:out] = fnames[:f].sub(/\.[^.]+$/, ".#{fnames[:ext]}")
164
+ isodoc_options = get_isodoc_options(file, options, ext)
165
+ thread = nil
166
+ unless process_ext_simple(ext, isodoc, fnames, options,
167
+ isodoc_options)
168
+ thread = process_exts1(ext, fnames, isodoc, options, isodoc_options)
169
+ end
170
+ thread
171
+ end
172
+
173
+ def process_ext_simple(ext, isodoc, fnames, options, isodoc_options)
174
+ if ext == :rxl
175
+ relaton_export(isodoc, options.merge(relaton: fnames[:out]))
176
+ elsif options[:passthrough_presentation_xml] && ext == :presentation
177
+ f = File.exists?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
178
+ FileUtils.cp f, fnames[:presentationxml]
179
+ elsif ext == :html && options[:sectionsplit]
180
+ sectionsplit_convert(fnames[:xml], isodoc, fnames[:out],
181
+ isodoc_options)
182
+ else return false
161
183
  end
184
+ true
162
185
  end
163
186
 
164
- def process_extensions1(ext, fnames, isodoc, isodoc_options)
187
+ def process_exts1(ext, fnames, isodoc, options, isodoc_options)
165
188
  if @processor.use_presentation_xml(ext)
166
- @processor.output(nil, fnames[:presentationxml], fnames[:out], ext,
167
- isodoc_options)
189
+ @queue.schedule(ext, fnames.dup, options.dup,
190
+ isodoc_options.dup) do |a, b, c, d|
191
+ process_output_threaded(a, b, c, d)
192
+ end
168
193
  else
169
- @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
170
- isodoc_options)
194
+ process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
171
195
  end
196
+ end
197
+
198
+ def process_output_threaded(ext, fnames1, options1, isodoc_options1)
199
+ @processor.output(nil, fnames1[:presentationxml], fnames1[:out], ext,
200
+ isodoc_options1)
201
+ wrap_html(options1, fnames1[:ext], fnames1[:out])
202
+ rescue StandardError => e
203
+ isodoc_error_process(e)
204
+ end
205
+
206
+ def process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
207
+ @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
208
+ isodoc_options)
209
+ nil # return as Thread
172
210
  rescue StandardError => e
173
211
  isodoc_error_process(e)
174
212
  end
@@ -184,15 +222,17 @@ module Metanorma
184
222
  end
185
223
  end
186
224
 
187
- def get_isodoc_options(file, options, _ext)
188
- isodoc_options = @processor.extract_options(file)
189
- isodoc_options[:datauriimage] = true if options[:datauriimage]
190
- isodoc_options[:sourcefilename] = options[:filename]
191
- %i(bare sectionsplit no_install_fonts baseassetpath aligncrosselements)
192
- .each do |x|
193
- isodoc_options[x] ||= options[x]
194
- end
195
- isodoc_options
225
+ def get_isodoc_options(file, options, ext)
226
+ ret = @processor.extract_options(file)
227
+ ret[:datauriimage] = true if options[:datauriimage]
228
+ ret[:sourcefilename] = options[:filename]
229
+ %i(bare sectionsplit no_install_fonts baseassetpath aligncrosselements
230
+ tocfigures toctables tocrecommendations)
231
+ .each { |x| ret[x] ||= options[x] }
232
+ ext == :pdf && FontistUtils.has_fonts_manifest?(@processor, options) and
233
+ ret[:mn2pdf] =
234
+ { font_manifest: FontistUtils.location_manifest(@processor) }
235
+ ret
196
236
  end
197
237
 
198
238
  # @param options [Hash]
@@ -68,6 +68,7 @@ module Metanorma
68
68
  pdf-owner-password pdf-allow-copy-content pdf-allow-edit-content
69
69
  pdf-allow-assemble-document pdf-allow-edit-annotations
70
70
  pdf-allow-print pdf-allow-print-hq pdf-allow-fill-in-forms
71
+ toc-figures toc-tables toc-recommendations
71
72
  pdf-allow-access-content pdf-encrypt-metadata).freeze
72
73
 
73
74
  def extract_options(file)
@@ -1,7 +1,7 @@
1
1
  # Registry of all Metanorma types and entry points
2
2
  #
3
3
 
4
- require 'singleton'
4
+ require "singleton"
5
5
 
6
6
  class Error < StandardError
7
7
  end
@@ -14,15 +14,16 @@ module Metanorma
14
14
 
15
15
  def initialize
16
16
  @processors = {}
17
- @aliases = {csd: :cc, m3d: :m3aawg, mpfd: :mpfa, csand: :csa}
17
+ @aliases = { csd: :cc, m3d: :m3aawg, mpfd: :mpfa, csand: :csa }
18
18
  end
19
19
 
20
20
  def alias(x)
21
21
  @aliases[x]
22
22
  end
23
23
 
24
- def register processor
24
+ def register(processor)
25
25
  raise Error unless processor < ::Metanorma::Processor
26
+
26
27
  p = processor.new
27
28
  # p.short[-1] is the canonical name
28
29
  short = Array(p.short)
@@ -31,7 +32,8 @@ module Metanorma
31
32
  @aliases[s] = short[-1]
32
33
  end
33
34
  Array(p.short)
34
- Util.log("[metanorma] processor \"#{Array(p.short)[0]}\" registered", :info)
35
+ Util.log("[metanorma] processor \"#{Array(p.short)[0]}\" registered",
36
+ :info)
35
37
  end
36
38
 
37
39
  def find_processor(short)
@@ -42,21 +44,17 @@ module Metanorma
42
44
  @processors.keys
43
45
  end
44
46
 
45
- def processors
46
- @processors
47
- end
48
-
49
47
  def output_formats
50
- @processors.inject({}) do |acc, (k,v)|
48
+ @processors.inject({}) do |acc, (k, v)|
51
49
  acc[k] = v.output_formats
52
50
  acc
53
51
  end
54
52
  end
55
53
 
56
54
  def root_tags
57
- @processors.inject({}) do |acc, (k,v)|
55
+ @processors.inject({}) do |acc, (k, v)|
58
56
  if v.asciidoctor_backend
59
- x = Asciidoctor.load nil, {backend: v.asciidoctor_backend}
57
+ x = Asciidoctor.load nil, { backend: v.asciidoctor_backend }
60
58
  acc[k] = x.converter.xml_root_tag
61
59
  end
62
60
  acc
@@ -1,3 +1,3 @@
1
1
  module Metanorma
2
- VERSION = "1.4.3".freeze
2
+ VERSION = "1.4.6".freeze
3
3
  end
@@ -0,0 +1,29 @@
1
+ module Metanorma
2
+ class WorkersPool
3
+ def initialize(workers)
4
+ @workers = workers
5
+ @queue = SizedQueue.new(@workers)
6
+ @threads = Array.new(@workers) do
7
+ Thread.new do
8
+ catch(:exit) do
9
+ loop do
10
+ job, args = @queue.pop
11
+ job.call *args
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
17
+
18
+ def schedule(*args, &block)
19
+ @queue << [block, args]
20
+ end
21
+
22
+ def shutdown
23
+ @workers.times do
24
+ schedule { throw :exit }
25
+ end
26
+ @threads.map(&:join)
27
+ end
28
+ end
29
+ end
data/metanorma.gemspec CHANGED
@@ -36,12 +36,11 @@ Gem::Specification.new do |spec|
36
36
 
37
37
  spec.add_development_dependency "debug"
38
38
  spec.add_development_dependency "equivalent-xml", "~> 0.6"
39
- spec.add_development_dependency "metanorma-iso", "~> 1.10"
39
+ spec.add_development_dependency "metanorma-iso", "~> 2.0"
40
40
  spec.add_development_dependency "mnconvert"
41
41
  spec.add_development_dependency "rake", "~> 13.0"
42
42
  spec.add_development_dependency "rspec", "~> 3.0"
43
43
  spec.add_development_dependency "rspec-command", "~> 1.0"
44
44
  spec.add_development_dependency "rubocop", "~> 1.5.2"
45
45
  spec.add_development_dependency "sassc", "~> 2.4.0"
46
- spec.add_development_dependency "reline", "~> 0.2.8.pre.11"
47
46
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: metanorma
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.4.3
4
+ version: 1.4.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-12-13 00:00:00.000000000 Z
11
+ date: 2022-03-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: asciidoctor
@@ -142,14 +142,14 @@ dependencies:
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: '1.10'
145
+ version: '2.0'
146
146
  type: :development
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: '1.10'
152
+ version: '2.0'
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: mnconvert
155
155
  requirement: !ruby/object:Gem::Requirement
@@ -234,20 +234,6 @@ dependencies:
234
234
  - - "~>"
235
235
  - !ruby/object:Gem::Version
236
236
  version: 2.4.0
237
- - !ruby/object:Gem::Dependency
238
- name: reline
239
- requirement: !ruby/object:Gem::Requirement
240
- requirements:
241
- - - "~>"
242
- - !ruby/object:Gem::Version
243
- version: 0.2.8.pre.11
244
- type: :development
245
- prerelease: false
246
- version_requirements: !ruby/object:Gem::Requirement
247
- requirements:
248
- - - "~>"
249
- - !ruby/object:Gem::Version
250
- version: 0.2.8.pre.11
251
237
  description: Library to process any Metanorma standard.
252
238
  email:
253
239
  - open.source@ribose.com
@@ -298,6 +284,7 @@ files:
298
284
  - lib/metanorma/sectionsplit.rb
299
285
  - lib/metanorma/util.rb
300
286
  - lib/metanorma/version.rb
287
+ - lib/metanorma/worker_pool.rb
301
288
  - metanorma.gemspec
302
289
  homepage: https://github.com/metanorma/metanorma
303
290
  licenses: