metanorma 1.4.0 → 1.4.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cebdc5e3a04db33d1bf3c0042d914f03a1e7c36ae67d625138fb1499679abdcf
4
- data.tar.gz: 82f7a4b816cea046181cfd05da94ca21a52f1c12ac02d9f6f2e89ae1b07e6f60
3
+ metadata.gz: c8457479b196c4d3db250ce33f650cd822f1ac8f1a510d1b6ac9653038446e1e
4
+ data.tar.gz: f1219e134ab2113cd3e25c5bca39fe3194a3a038f9b6500f0d5e35d66f168d0d
5
5
  SHA512:
6
- metadata.gz: cfa8565f3b0a55f12f0d0e8971d82c8bf619390cfc76860514dda7b1801267d7e04719161048ff7d9b16a71e238df2e24e145198229b5b5930233ae9110933ec
7
- data.tar.gz: 81764ee18d5c1b36f3aacf19c0a4cf4b02cd4fb0a8979fe9d0a6733a93f4f05ddc845be35b25ec38be31d595fd04a019ae6aaf6ca33a97664519550e1abe280b
6
+ metadata.gz: 8e62d329f40ca168b5f2250ae86e676b89650c754233a4d249eea45bdf7652863ce5824a128bbff0c6225435a97be4bb8b420ce9cfca44587be9d1d852c33599
7
+ data.tar.gz: 1b18636330ca30bba9f2a0080e1728c6d05763349ffafebab78e1b61043f3b45c5160156dfdbf541d886fc14e195ee54808cbfbc4dd8d296e7bb56ee5edca735
data/Gemfile CHANGED
@@ -9,3 +9,4 @@ gemspec
9
9
  if File.exist? "Gemfile.devel"
10
10
  eval File.read("Gemfile.devel"), nil, "Gemfile.devel" # rubocop:disable Security/Eval
11
11
  end
12
+
data/README.adoc CHANGED
@@ -247,6 +247,13 @@ The options hash has the same structure it does when invoked in metanorma-cli:
247
247
  `:relaton`: exports the bibdata Relaton XML description of the document (which is part of its Metanorma XML)
248
248
  to the nominated directory
249
249
 
250
+ == Threaded execution
251
+
252
+ Metanorma has threaded execution, to generate output documents from the same Presentation XML input more quickly.
253
+ Similar to https://github.com/relaton/relaton[relaton], the `METANORMA_PARALLEL` environment variable
254
+ can be used to override the default number of parallel fetches used.
255
+
256
+
250
257
  == Origin of name
251
258
 
252
259
  *Meta-* is a prefix of Greek origin ("μετα") for "`with`" "`after`".
@@ -91,12 +91,9 @@ module Metanorma
91
91
  private
92
92
 
93
93
  def parse_xml(file)
94
- xml = Nokogiri::XML File.read(file, encoding: "UTF-8") do |config|
95
- config.huge
96
- end
97
- if (b = xml.at("/xmlns:metanorma-collection/xmlns:bibdata"))
98
- bd = Relaton::Cli.parse_xml b
99
- end
94
+ xml = Nokogiri::XML File.read(file, encoding: "UTF-8") { |c| c.huge }
95
+ (b = xml.at("/xmlns:metanorma-collection/xmlns:bibdata")) and
96
+ bd = Relaton::Cli.parse_xml(b)
100
97
  mnf_xml = xml.at("/xmlns:metanorma-collection/xmlns:manifest")
101
98
  mnf = CollectionManifest.from_xml mnf_xml
102
99
  pref = pref_final_content xml.at("//xmlns:prefatory-content")
@@ -83,11 +83,15 @@ module Metanorma
83
83
  add_document_suffix(identifier, docxml)
84
84
  update_direct_refs_to_docs(docxml, identifier)
85
85
  svgmap_resolve(datauri_encode(docxml))
86
+ hide_refs(docxml)
87
+ docxml.to_xml
88
+ end
89
+
90
+ def hide_refs(docxml)
86
91
  docxml.xpath(ns("//references[bibitem][not(./bibitem[not(@hidden) or "\
87
92
  "@hidden = 'false'])]")).each do |f|
88
93
  f["hidden"] = "true"
89
94
  end
90
- docxml.to_xml
91
95
  end
92
96
 
93
97
  def supply_repo_ids(docxml)
@@ -97,7 +101,8 @@ module Metanorma
97
101
  b.xpath(ns("./docidentifier")).each do |d|
98
102
  next unless @files[d.text]
99
103
 
100
- d.next = "<docidentifier type='repository'>current-metanorma-collection/#{d.text}"
104
+ d.next = "<docidentifier type='repository'>"\
105
+ "current-metanorma-collection/#{d.text}"
101
106
  end
102
107
  end
103
108
  end
@@ -111,29 +116,30 @@ module Metanorma
111
116
 
112
117
  def svgmap_resolve(docxml)
113
118
  isodoc = IsoDoc::Convert.new({})
119
+ isodoc.bibitem_lookup(docxml)
114
120
  docxml.xpath(ns("//svgmap//eref")).each do |e|
115
- href = isodoc.eref_target(e)
116
- next if href == "##{e['bibitemid']}" ||
117
- href =~ /^#/ && !docxml.at("//*[@id = '#{href.sub(/^#/, '')}']")
118
-
119
- e["target"] = href.strip
120
- e.name = "link"
121
- e&.elements&.remove
121
+ svgmap_resolve1(e, isodoc)
122
122
  end
123
123
  Metanorma::Utils::svgmap_rewrite(docxml, "")
124
124
  end
125
125
 
126
+ def svgmap_resolve1(eref, isodoc)
127
+ href = isodoc.eref_target(eref)
128
+ return if href == "##{eref['bibitemid']}" ||
129
+ (href =~ /^#/ && !docxml.at("//*[@id = '#{href.sub(/^#/, '')}']"))
130
+
131
+ eref["target"] = href.strip
132
+ eref.name = "link"
133
+ eref&.elements&.remove
134
+ end
135
+
126
136
  # repo(current-metanorma-collection/ISO 17301-1:2016)
127
137
  # replaced by bibdata of "ISO 17301-1:2016" in situ as bibitem.
128
138
  # Any erefs to that bibitem id are replaced with relative URL
129
139
  # Preferably with anchor, and is a job to realise dynamic lookup
130
140
  # of localities.
131
141
  def update_direct_refs_to_docs(docxml, identifier)
132
- erefs = docxml.xpath(ns("//eref"))
133
- .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
134
- m[:citeas][i["citeas"]] = true
135
- m[:bibitemid][i["bibitemid"]] = true
136
- end
142
+ erefs = collect_erefs(docxml)
137
143
  docxml.xpath(ns("//bibitem[not(ancestor::bibitem)]")).each do |b|
138
144
  docid = b&.at(ns("./docidentifier[@type = 'repository']"))&.text
139
145
  next unless docid && %r{^current-metanorma-collection/}.match(docid)
@@ -144,6 +150,14 @@ module Metanorma
144
150
  end
145
151
  end
146
152
 
153
+ def collect_erefs(docxml)
154
+ docxml.xpath(ns("//eref"))
155
+ .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
156
+ m[:citeas][i["citeas"]] = true
157
+ m[:bibitemid][i["bibitemid"]] = true
158
+ end
159
+ end
160
+
147
161
  # Resolve erefs to a container of ids in another doc,
148
162
  # to an anchor eref (direct link)
149
163
  def update_indirect_refs_to_docs(docxml, internal_refs)
@@ -99,17 +99,15 @@ module Metanorma
99
99
  out = ref["attachment"] ? ref["fileref"] : File.basename(ref["fileref"])
100
100
  ret = if ref["fileref"]
101
101
  { type: "fileref", ref: @documents[identifier].file,
102
- rel_path: ref["fileref"],
103
- out_path: out }
104
- else
105
- { type: "id", ref: ref["id"] }
102
+ rel_path: ref["fileref"], out_path: out }
103
+ else { type: "id", ref: ref["id"] }
106
104
  end
107
105
  %i(attachment sectionsplit index).each do |s|
108
106
  ret[s] = ref[s.to_s] if ref[s.to_s]
109
107
  end
110
- ret[:presentationxml] = ref["presentation-xml"] if ref["presentation-xml"]
111
- ret[:bareafterfirst] = ref["bare-after-first"] if ref["bare-after-first"]
112
- ret
108
+ ret[:presentationxml] = ref["presentation-xml"]
109
+ ret[:bareafterfirst] = ref["bare-after-first"]
110
+ ret.compact
113
111
  end
114
112
 
115
113
  def add_suffix_to_attributes(doc, suffix, tag_name, attribute_name)
@@ -164,10 +162,15 @@ module Metanorma
164
162
  def file_compile(file, filename, identifier)
165
163
  return if @files[identifier][:sectionsplit] == "true"
166
164
 
167
- @compile.compile file.path, { format: :asciidoc, extension_keys: @format }
168
- .merge(compile_options(identifier))
165
+ opts = {
166
+ format: :asciidoc,
167
+ extension_keys: @format,
168
+ output_dir: @outdir,
169
+ }.merge(compile_options(identifier))
170
+
171
+ @compile.compile file, opts
169
172
  @files[identifier][:outputs] = {}
170
- file_compile_formats(file, filename, identifier)
173
+ file_compile_formats(filename, identifier)
171
174
  end
172
175
 
173
176
  def compile_options(identifier)
@@ -182,29 +185,18 @@ module Metanorma
182
185
  ret
183
186
  end
184
187
 
185
- def file_compile_formats(file, filename, identifier)
188
+ def file_compile_formats(filename, identifier)
189
+ file_id = @files[identifier]
190
+ @format << :presentation if @format.include?(:pdf)
186
191
  @format.each do |e|
187
192
  ext = @compile.processor.output_formats[e]
188
193
  fn = File.basename(filename).sub(/(?<=\.)[^.]+$/, ext.to_s)
189
- if /html$/.match?(ext) && @files[identifier][:sectionsplit]
190
- # file_sectionsplit_copy(file, fn, identifier, ext, e)
191
- else
192
- FileUtils.cp file.path.sub(/\.xml$/, ".#{ext}"),
193
- File.join(@outdir, fn)
194
- @files[identifier][:outputs][e] = File.join(@outdir, fn)
194
+ unless /html$/.match?(ext) && file_id[:sectionsplit]
195
+ file_id[:outputs][e] = File.join(@outdir, fn)
195
196
  end
196
197
  end
197
198
  end
198
199
 
199
- def file_sectionsplit_copy(file, base, identifier, ext, format)
200
- dir = file.path.sub(/\.xml$/, ".#{ext}_collection")
201
- files = Dir.glob("#{dir}/*.#{ext}")
202
- FileUtils.cp files, @outdir
203
- cover = File.join(@outdir, base.sub(/\.html$/, ".index.html"))
204
- FileUtils.cp File.join(dir, "index.html"), cover
205
- @files[identifier][:outputs][format] = cover
206
- end
207
-
208
200
  def copy_file_to_dest(fileref)
209
201
  dest = File.join(@outdir, fileref[:out_path])
210
202
  FileUtils.mkdir_p(File.dirname(dest))
@@ -223,12 +215,13 @@ module Metanorma
223
215
  else
224
216
  file, filename = targetfile(x, read: true)
225
217
  warn "\n\n\n\n\nProcess #{filename}: #{DateTime.now.strftime('%H:%M:%S')}"
226
- file = update_xrefs(file, identifier, internal_refs)
227
- Tempfile.open(["collection", ".xml"], encoding: "utf-8") do |f|
228
- f.write(file)
229
- f.close
230
- file_compile(f, filename, identifier)
231
- end
218
+ collection_xml = update_xrefs(file, identifier, internal_refs)
219
+ collection_filename = File.basename(filename, File.extname(filename))
220
+ collection_xml_path = File.join(Dir.tmpdir,
221
+ "#{collection_filename}.xml")
222
+ File.write collection_xml_path, collection_xml, encoding: "UTF-8"
223
+ file_compile(collection_xml_path, filename, identifier)
224
+ FileUtils.rm(collection_xml_path)
232
225
  end
233
226
  end
234
227
  end
@@ -16,7 +16,7 @@ module Metanorma
16
16
  # @param options [Hash]
17
17
  # @option options [String] :coverpage cover page HTML (Liquid template)
18
18
  # @option options [Array<Symbol>] :format list of formats (xml,html,doc,pdf)
19
- # @option options [String] :ourput_folder output directory
19
+ # @option options [String] :output_folder output directory
20
20
  #
21
21
  # We presuppose that the bibdata of the document is equivalent to that of
22
22
  # the collection, and that the flavour gem can sensibly process it. We may
@@ -62,7 +62,6 @@ module Metanorma
62
62
  # @option options [Strong] :ourput_folder output directory
63
63
  def self.render(col, options = {})
64
64
  folder = File.dirname col.file
65
- # require "byebug"; byebug
66
65
  warn "\n\n\n\n\nRender Init: #{DateTime.now.strftime('%H:%M:%S')}"
67
66
  cr = new(col, folder, options)
68
67
  warn "\n\n\n\n\nRender Files: #{DateTime.now.strftime('%H:%M:%S')}"
@@ -8,6 +8,8 @@ require_relative "compile_validate"
8
8
  require_relative "fontist_utils"
9
9
  require_relative "util"
10
10
  require_relative "sectionsplit"
11
+ require_relative "extract"
12
+ require_relative "worker_pool"
11
13
 
12
14
  module Metanorma
13
15
  class Compile
@@ -32,7 +34,7 @@ module Metanorma
32
34
  extract(isodoc, options[:extract], options[:extract_type])
33
35
  FontistUtils.install_fonts(@processor, options) unless @fontist_installed
34
36
  @fontist_installed = true
35
- process_extensions(extensions, file, isodoc, options)
37
+ process_exts(filename, extensions, file, isodoc, options)
36
38
  end
37
39
 
38
40
  def require_libraries(options)
@@ -128,112 +130,85 @@ module Metanorma
128
130
  File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
129
131
  end
130
132
 
131
- def clean_sourcecode(xml)
132
- xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
133
- ".//xmlns:annotation").each(&:remove)
134
- xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
135
- HTMLEntities.new.decode(xml.children.to_xml)
133
+ def export_output(fname, content, **options)
134
+ mode = options[:binary] ? "wb" : "w:UTF-8"
135
+ File.open(fname, mode) { |f| f.write content }
136
136
  end
137
137
 
138
- def extract(isodoc, dirname, extract_types)
139
- return unless dirname
140
-
141
- if extract_types.nil? || extract_types.empty?
142
- extract_types = %i[sourcecode image requirement]
138
+ def wrap_html(options, file_extension, outfilename)
139
+ if options[:wrapper] && /html$/.match(file_extension)
140
+ outfilename = outfilename.sub(/\.html$/, "")
141
+ FileUtils.mkdir_p outfilename
142
+ FileUtils.mv "#{outfilename}.html", outfilename
143
+ FileUtils.mv "#{outfilename}_images", outfilename, force: true
143
144
  end
144
- FileUtils.rm_rf dirname
145
- FileUtils.mkdir_p dirname
146
- xml = Nokogiri::XML(isodoc) { |config| config.huge }
147
- sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
148
- image_export(xml, dirname) if extract_types.include? :image
149
- requirement_export(xml, dirname) if extract_types.include? :requirement
150
145
  end
151
146
 
152
- def sourcecode_export(xml, dirname)
153
- xml.at("//sourcecode | //xmlns:sourcecode") or return
154
- FileUtils.mkdir_p "#{dirname}/sourcecode"
155
- xml.xpath("//sourcecode | //xmlns:sourcecode").each_with_index do |s, i|
156
- filename = s["filename"] || sprintf("sourcecode-%04d.txt", i)
157
- File.open("#{dirname}/sourcecode/#{filename}", "w:UTF-8") do |f|
158
- f.write clean_sourcecode(s.dup)
159
- end
147
+ # isodoc is Raw Metanorma XML
148
+ def process_exts(filename, extensions, file, isodoc, options)
149
+ f = File.expand_path(change_output_dir(options))
150
+ fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
151
+ orig_filename: File.expand_path(filename),
152
+ presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
153
+ @queue = ::Metanorma::WorkersPool
154
+ .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
155
+ Util.sort_extensions_execution(extensions).each do |ext|
156
+ process_ext(ext, file, isodoc, fnames, options)
160
157
  end
158
+ @queue.shutdown
161
159
  end
162
160
 
163
- def image_export(xml, dirname)
164
- xml.at("//image | //xmlns:image") or return
165
- FileUtils.mkdir_p "#{dirname}/image"
166
- xml.xpath("//image | //xmlns:image").each_with_index do |s, i|
167
- next unless /^data:image/.match? s["src"]
168
-
169
- %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ s["src"]
170
- filename = s["filename"] || sprintf("image-%04d.%s", i, imgtype)
171
- File.open("#{dirname}/image/#{filename}", "wb") do |f|
172
- f.write(Base64.strict_decode64(imgdata))
173
- end
161
+ def process_ext(ext, file, isodoc, fnames, options)
162
+ fnames[:ext] = @processor.output_formats[ext]
163
+ fnames[:out] = fnames[:f].sub(/\.[^.]+$/, ".#{fnames[:ext]}")
164
+ isodoc_options = get_isodoc_options(file, options, ext)
165
+ thread = nil
166
+ unless process_ext_simple(ext, isodoc, fnames, options,
167
+ isodoc_options)
168
+ thread = process_exts1(ext, fnames, isodoc, options, isodoc_options)
174
169
  end
170
+ thread
171
+ end
172
+
173
+ def process_ext_simple(ext, isodoc, fnames, options, isodoc_options)
174
+ if ext == :rxl
175
+ relaton_export(isodoc, options.merge(relaton: fnames[:out]))
176
+ elsif options[:passthrough_presentation_xml] && ext == :presentation
177
+ f = File.exists?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
178
+ FileUtils.cp f, fnames[:presentationxml]
179
+ elsif ext == :html && options[:sectionsplit]
180
+ sectionsplit_convert(fnames[:xml], isodoc, fnames[:out],
181
+ isodoc_options)
182
+ else return false
183
+ end
184
+ true
175
185
  end
176
186
 
177
- REQUIREMENT_XPATH = "//requirement | //xmlns:requirement | "\
178
- "//recommendation | //xmlns:recommendation | //permission | "\
179
- "//xmlns:permission".freeze
180
-
181
- def requirement_export(xml, dirname)
182
- xml.at(REQUIREMENT_XPATH) or return
183
- FileUtils.mkdir_p "#{dirname}/requirement"
184
- xml.xpath(REQUIREMENT_XPATH).each_with_index do |s, i|
185
- filename = s["filename"] || sprintf("%s-%04d.xml", s.name, i)
186
- File.open("#{dirname}/requirement/#{filename}", "w:UTF-8") do |f|
187
- f.write s
187
+ def process_exts1(ext, fnames, isodoc, options, isodoc_options)
188
+ if @processor.use_presentation_xml(ext)
189
+ @queue.schedule(ext, fnames.dup, options.dup,
190
+ isodoc_options.dup) do |a, b, c, d|
191
+ process_output_threaded(a, b, c, d)
188
192
  end
193
+ else
194
+ process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
189
195
  end
190
196
  end
191
197
 
192
- def wrap_html(options, file_extension, outfilename)
193
- if options[:wrapper] && /html$/.match(file_extension)
194
- outfilename = outfilename.sub(/\.html$/, "")
195
- FileUtils.mkdir_p outfilename
196
- FileUtils.mv "#{outfilename}.html", outfilename
197
- FileUtils.mv "#{outfilename}_images", outfilename, force: true
198
- end
198
+ def process_output_threaded(ext, fnames1, options1, isodoc_options1)
199
+ @processor.output(nil, fnames1[:presentationxml], fnames1[:out], ext,
200
+ isodoc_options1)
201
+ wrap_html(options1, fnames1[:ext], fnames1[:out])
202
+ rescue StandardError => e
203
+ isodoc_error_process(e)
199
204
  end
200
205
 
201
- # isodoc is Raw Metanorma XML
202
- def process_extensions(extensions, file, isodoc, options)
203
- f = change_output_dir options
204
- xml_name = f.sub(/\.[^.]+$/, ".xml")
205
- presentationxml_name = f.sub(/\.[^.]+$/, ".presentation.xml")
206
- Util.sort_extensions_execution(extensions).each do |ext|
207
- file_extension = @processor.output_formats[ext]
208
- outfilename = f.sub(/\.[^.]+$/, ".#{file_extension}")
209
- isodoc_options = get_isodoc_options(file, options, ext)
210
- if ext == :rxl
211
- relaton_export(isodoc, options.merge(relaton: outfilename))
212
- elsif options[:passthrough_presentation_xml] && ext == :presentation
213
- FileUtils.cp f, presentationxml_name
214
- elsif ext == :html && options[:sectionsplit]
215
- sectionsplit_convert(xml_name, isodoc, outfilename, isodoc_options)
216
- else
217
- if ext == :pdf && FontistUtils.has_fonts_manifest?(@processor,
218
- options)
219
- isodoc_options[:mn2pdf] = {
220
- font_manifest: FontistUtils.location_manifest(@processor),
221
- }
222
- end
223
- begin
224
- if @processor.use_presentation_xml(ext)
225
- @processor.output(nil, presentationxml_name, outfilename, ext,
226
- isodoc_options)
227
- else
228
- @processor.output(isodoc, xml_name, outfilename, ext,
229
- isodoc_options)
230
- end
231
- rescue StandardError => e
232
- isodoc_error_process(e)
233
- end
234
- end
235
- wrap_html(options, file_extension, outfilename)
236
- end
206
+ def process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
207
+ @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
208
+ isodoc_options)
209
+ nil # return as Thread
210
+ rescue StandardError => e
211
+ isodoc_error_process(e)
237
212
  end
238
213
 
239
214
  private
@@ -248,14 +223,15 @@ module Metanorma
248
223
  end
249
224
 
250
225
  def get_isodoc_options(file, options, ext)
251
- isodoc_options = @processor.extract_options(file)
252
- isodoc_options[:datauriimage] = true if options[:datauriimage]
253
- isodoc_options[:sourcefilename] = options[:filename]
226
+ ret = @processor.extract_options(file)
227
+ ret[:datauriimage] = true if options[:datauriimage]
228
+ ret[:sourcefilename] = options[:filename]
254
229
  %i(bare sectionsplit no_install_fonts baseassetpath aligncrosselements)
255
- .each do |x|
256
- isodoc_options[x] ||= options[x]
257
- end
258
- isodoc_options
230
+ .each { |x| ret[x] ||= options[x] }
231
+ ext == :pdf && FontistUtils.has_fonts_manifest?(@processor, options) and
232
+ ret[:mn2pdf] =
233
+ { font_manifest: FontistUtils.location_manifest(@processor) }
234
+ ret
259
235
  end
260
236
 
261
237
  # @param options [Hash]
@@ -0,0 +1,72 @@
1
+ module Metanorma
2
+ class Compile
3
+ def relaton_export(isodoc, options)
4
+ return unless options[:relaton]
5
+
6
+ xml = Nokogiri::XML(isodoc) { |config| config.huge }
7
+ bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
8
+ # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
9
+ # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
10
+ File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
11
+ end
12
+
13
+ def clean_sourcecode(xml)
14
+ xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
15
+ ".//xmlns:annotation").each(&:remove)
16
+ xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
17
+ HTMLEntities.new.decode(xml.children.to_xml)
18
+ end
19
+
20
+ def extract(isodoc, dirname, extract_types)
21
+ return unless dirname
22
+
23
+ extract_types.nil? || extract_types.empty? and
24
+ extract_types = %i[sourcecode image requirement]
25
+ FileUtils.rm_rf dirname
26
+ FileUtils.mkdir_p dirname
27
+ xml = Nokogiri::XML(isodoc) { |config| config.huge }
28
+ sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
29
+ image_export(xml, dirname) if extract_types.include? :image
30
+ extract_types.include?(:requirement) and
31
+ requirement_export(xml, dirname)
32
+ end
33
+
34
+ def sourcecode_export(xml, dirname)
35
+ xml.at("//sourcecode | //xmlns:sourcecode") or return
36
+ FileUtils.mkdir_p "#{dirname}/sourcecode"
37
+ xml.xpath("//sourcecode | //xmlns:sourcecode").each_with_index do |s, i|
38
+ filename = s["filename"] || sprintf("sourcecode-%04d.txt", i)
39
+ export_output("#{dirname}/sourcecode/#{filename}",
40
+ clean_sourcecode(s.dup))
41
+ end
42
+ end
43
+
44
+ def image_export(xml, dirname)
45
+ xml.at("//image | //xmlns:image") or return
46
+ FileUtils.mkdir_p "#{dirname}/image"
47
+ xml.xpath("//image | //xmlns:image").each_with_index do |s, i|
48
+ next unless /^data:image/.match? s["src"]
49
+
50
+ %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ s["src"]
51
+ fn = s["filename"] || sprintf("image-%<num>04d.%<name>s",
52
+ num: i, name: imgtype)
53
+ export_output("#{dirname}/image/#{fn}", Base64.strict_decode64(imgdata),
54
+ binary: true)
55
+ end
56
+ end
57
+
58
+ REQUIREMENT_XPATH =
59
+ "//requirement | //xmlns:requirement | //recommendation | "\
60
+ "//xmlns:recommendation | //permission | //xmlns:permission".freeze
61
+
62
+ def requirement_export(xml, dirname)
63
+ xml.at(REQUIREMENT_XPATH) or return
64
+ FileUtils.mkdir_p "#{dirname}/requirement"
65
+ xml.xpath(REQUIREMENT_XPATH).each_with_index do |s, i|
66
+ fn = s["filename"] ||
67
+ sprintf("%<name>s-%<num>04d.xml", name: s.name, num: i)
68
+ export_output("#{dirname}/requirement/#{fn}", s)
69
+ end
70
+ end
71
+ end
72
+ end
@@ -56,13 +56,19 @@ module Metanorma
56
56
  attr&.sub(/^#{name}:\s*$/, "#{name}: true")&.sub(/^#{name}:\s+/, "")
57
57
  end
58
58
 
59
- ADOC_OPTIONS = %w(htmlstylesheet htmlcoverpage htmlintropage scripts
60
- scripts-override scripts-pdf wordstylesheet i18nyaml
61
- standardstylesheet header wordcoverpage wordintropage
62
- ulstyle olstyle htmlstylesheet-override bare
63
- htmltoclevels doctoclevels sectionsplit base-asset-path
64
- body-font header-font monospace-font title-font
65
- align-cross-elements wordstylesheet-override).freeze
59
+ ADOC_OPTIONS =
60
+ %w(htmlstylesheet htmlcoverpage htmlintropage scripts
61
+ scripts-override scripts-pdf wordstylesheet i18nyaml
62
+ standardstylesheet header wordcoverpage wordintropage
63
+ ulstyle olstyle htmlstylesheet-override bare
64
+ htmltoclevels doctoclevels sectionsplit base-asset-path
65
+ body-font header-font monospace-font title-font
66
+ align-cross-elements wordstylesheet-override
67
+ pdf-encrypt pdf-encryption-length pdf-user-password
68
+ pdf-owner-password pdf-allow-copy-content pdf-allow-edit-content
69
+ pdf-allow-assemble-document pdf-allow-edit-annotations
70
+ pdf-allow-print pdf-allow-print-hq pdf-allow-fill-in-forms
71
+ pdf-allow-access-content pdf-encrypt-metadata).freeze
66
72
 
67
73
  def extract_options(file)
68
74
  header = file.sub(/\n\n.*$/m, "\n")
@@ -1,7 +1,7 @@
1
1
  # Registry of all Metanorma types and entry points
2
2
  #
3
3
 
4
- require 'singleton'
4
+ require "singleton"
5
5
 
6
6
  class Error < StandardError
7
7
  end
@@ -14,15 +14,16 @@ module Metanorma
14
14
 
15
15
  def initialize
16
16
  @processors = {}
17
- @aliases = {csd: :cc, m3d: :m3aawg, mpfd: :mpfa, csand: :csa}
17
+ @aliases = { csd: :cc, m3d: :m3aawg, mpfd: :mpfa, csand: :csa }
18
18
  end
19
19
 
20
20
  def alias(x)
21
21
  @aliases[x]
22
22
  end
23
23
 
24
- def register processor
24
+ def register(processor)
25
25
  raise Error unless processor < ::Metanorma::Processor
26
+
26
27
  p = processor.new
27
28
  # p.short[-1] is the canonical name
28
29
  short = Array(p.short)
@@ -31,7 +32,8 @@ module Metanorma
31
32
  @aliases[s] = short[-1]
32
33
  end
33
34
  Array(p.short)
34
- Util.log("[metanorma] processor \"#{Array(p.short)[0]}\" registered", :info)
35
+ Util.log("[metanorma] processor \"#{Array(p.short)[0]}\" registered",
36
+ :info)
35
37
  end
36
38
 
37
39
  def find_processor(short)
@@ -42,21 +44,17 @@ module Metanorma
42
44
  @processors.keys
43
45
  end
44
46
 
45
- def processors
46
- @processors
47
- end
48
-
49
47
  def output_formats
50
- @processors.inject({}) do |acc, (k,v)|
48
+ @processors.inject({}) do |acc, (k, v)|
51
49
  acc[k] = v.output_formats
52
50
  acc
53
51
  end
54
52
  end
55
53
 
56
54
  def root_tags
57
- @processors.inject({}) do |acc, (k,v)|
55
+ @processors.inject({}) do |acc, (k, v)|
58
56
  if v.asciidoctor_backend
59
- x = Asciidoctor.load nil, {backend: v.asciidoctor_backend}
57
+ x = Asciidoctor.load nil, { backend: v.asciidoctor_backend }
60
58
  acc[k] = x.converter.xml_root_tag
61
59
  end
62
60
  acc
@@ -1,3 +1,3 @@
1
1
  module Metanorma
2
- VERSION = "1.4.0".freeze
2
+ VERSION = "1.4.4".freeze
3
3
  end
@@ -0,0 +1,29 @@
1
+ module Metanorma
2
+ class WorkersPool
3
+ def initialize(workers)
4
+ @workers = workers
5
+ @queue = SizedQueue.new(@workers)
6
+ @threads = Array.new(@workers) do
7
+ Thread.new do
8
+ catch(:exit) do
9
+ loop do
10
+ job, args = @queue.pop
11
+ job.call *args
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
17
+
18
+ def schedule(*args, &block)
19
+ @queue << [block, args]
20
+ end
21
+
22
+ def shutdown
23
+ @workers.times do
24
+ schedule { throw :exit }
25
+ end
26
+ @threads.map(&:join)
27
+ end
28
+ end
29
+ end
data/metanorma.gemspec CHANGED
@@ -34,13 +34,14 @@ Gem::Specification.new do |spec|
34
34
  # spec.add_dependency "relaton-cli"
35
35
  # spec.add_dependency "metanorma-standoc"
36
36
 
37
- spec.add_development_dependency "byebug", "~> 10.0"
37
+ spec.add_development_dependency "debug"
38
38
  spec.add_development_dependency "equivalent-xml", "~> 0.6"
39
- spec.add_development_dependency "metanorma-iso", "~> 1.9.0"
39
+ spec.add_development_dependency "metanorma-iso", "~> 1.10"
40
+ spec.add_development_dependency "mnconvert"
40
41
  spec.add_development_dependency "rake", "~> 13.0"
41
42
  spec.add_development_dependency "rspec", "~> 3.0"
42
43
  spec.add_development_dependency "rspec-command", "~> 1.0"
43
44
  spec.add_development_dependency "rubocop", "~> 1.5.2"
44
45
  spec.add_development_dependency "sassc", "~> 2.4.0"
45
- spec.add_development_dependency "mnconvert"
46
+ spec.add_development_dependency "reline", "~> 0.2.8.pre.11"
46
47
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: metanorma
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.4.0
4
+ version: 1.4.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-10-25 00:00:00.000000000 Z
11
+ date: 2021-12-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: asciidoctor
@@ -109,19 +109,19 @@ dependencies:
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
111
  - !ruby/object:Gem::Dependency
112
- name: byebug
112
+ name: debug
113
113
  requirement: !ruby/object:Gem::Requirement
114
114
  requirements:
115
- - - "~>"
115
+ - - ">="
116
116
  - !ruby/object:Gem::Version
117
- version: '10.0'
117
+ version: '0'
118
118
  type: :development
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
- - - "~>"
122
+ - - ">="
123
123
  - !ruby/object:Gem::Version
124
- version: '10.0'
124
+ version: '0'
125
125
  - !ruby/object:Gem::Dependency
126
126
  name: equivalent-xml
127
127
  requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,28 @@ dependencies:
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: 1.9.0
145
+ version: '1.10'
146
146
  type: :development
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: 1.9.0
152
+ version: '1.10'
153
+ - !ruby/object:Gem::Dependency
154
+ name: mnconvert
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ version: '0'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: '0'
153
167
  - !ruby/object:Gem::Dependency
154
168
  name: rake
155
169
  requirement: !ruby/object:Gem::Requirement
@@ -221,19 +235,19 @@ dependencies:
221
235
  - !ruby/object:Gem::Version
222
236
  version: 2.4.0
223
237
  - !ruby/object:Gem::Dependency
224
- name: mnconvert
238
+ name: reline
225
239
  requirement: !ruby/object:Gem::Requirement
226
240
  requirements:
227
- - - ">="
241
+ - - "~>"
228
242
  - !ruby/object:Gem::Version
229
- version: '0'
243
+ version: 0.2.8.pre.11
230
244
  type: :development
231
245
  prerelease: false
232
246
  version_requirements: !ruby/object:Gem::Requirement
233
247
  requirements:
234
- - - ">="
248
+ - - "~>"
235
249
  - !ruby/object:Gem::Version
236
- version: '0'
250
+ version: 0.2.8.pre.11
237
251
  description: Library to process any Metanorma standard.
238
252
  email:
239
253
  - open.source@ribose.com
@@ -274,6 +288,7 @@ files:
274
288
  - lib/metanorma/compile_validate.rb
275
289
  - lib/metanorma/config.rb
276
290
  - lib/metanorma/document.rb
291
+ - lib/metanorma/extract.rb
277
292
  - lib/metanorma/fontist_utils.rb
278
293
  - lib/metanorma/input.rb
279
294
  - lib/metanorma/input/asciidoc.rb
@@ -283,6 +298,7 @@ files:
283
298
  - lib/metanorma/sectionsplit.rb
284
299
  - lib/metanorma/util.rb
285
300
  - lib/metanorma/version.rb
301
+ - lib/metanorma/worker_pool.rb
286
302
  - metanorma.gemspec
287
303
  homepage: https://github.com/metanorma/metanorma
288
304
  licenses:
@@ -303,7 +319,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
303
319
  - !ruby/object:Gem::Version
304
320
  version: '0'
305
321
  requirements: []
306
- rubygems_version: 3.2.22
322
+ rubygems_version: 3.2.32
307
323
  signing_key:
308
324
  specification_version: 4
309
325
  summary: Metanorma is the standard of standards; the metanorma gem allows you to create