stepmod-utils 0.3.18 → 0.3.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/stepmod-extract-terms +5 -1
- data/lib/stepmod/utils/concept.rb +4 -0
- data/lib/stepmod/utils/terms_extractor.rb +151 -119
- data/lib/stepmod/utils/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0d50ed47ffc263b007aa9f2bda3bf5535d9cf84dbc56f1fc87c63d4255ed3f02
|
4
|
+
data.tar.gz: 3572d73a116953f337ac8add7bb47133f33ab3244daa6f3ee58e9901f26cb5f1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5263394029779a53cca0c8f41171eb5bbc8e2490de14041c4e6ae56f8880a0f4eff49c1235d43d69f0d34501e5ae372a4c77bbfeb459147765fd87f81014d7dd
|
7
|
+
data.tar.gz: 6aab6c46a9023e194fe278cedd353807d070eba8c61949777af13ff63a26d3f652a304575cfc56873b54fd57187347a1321df40d7527905320689664da89991c
|
data/exe/stepmod-extract-terms
CHANGED
@@ -153,10 +153,13 @@ end
|
|
153
153
|
|
154
154
|
log "INFO: written summary file to: 04x-stepmod-entities-resources.adoc"
|
155
155
|
|
156
|
-
part_modules.
|
156
|
+
part_modules.sort_by do |(bibdata, part_modules_arm, part_modules_mim)|
|
157
|
+
bibdata.part.to_i
|
158
|
+
end.each do |(bibdata, part_modules_arm, part_modules_mim)|
|
157
159
|
fn = "05x-stepmod-entities-modules-#{bibdata.part}.adoc"
|
158
160
|
File.open(fn, "w") do |file|
|
159
161
|
file.puts("")
|
162
|
+
|
160
163
|
unless part_modules_arm.empty?
|
161
164
|
schema_name = part_modules_arm.first.first
|
162
165
|
concepts = part_modules_arm.first.last.to_a.map do |n|
|
@@ -181,6 +184,7 @@ part_modules.each do |(bibdata, part_modules_arm, part_modules_mim)|
|
|
181
184
|
file.puts(replace_images(concepts.map(&:to_mn_adoc).join("\n")))
|
182
185
|
end
|
183
186
|
end
|
187
|
+
|
184
188
|
log "INFO: written to: #{fn}"
|
185
189
|
end
|
186
190
|
|
@@ -10,6 +10,8 @@ module Stepmod
|
|
10
10
|
file_path
|
11
11
|
)
|
12
12
|
|
13
|
+
# TODO: converted_definition is not supposed to be an attribute, it is
|
14
|
+
# supposed to be a method!
|
13
15
|
class << self
|
14
16
|
def parse(definition_xml, reference_anchor:, reference_clause:, file_path:, language_code: "en")
|
15
17
|
converted_definition = Stepmod::Utils::StepmodDefinitionConverter.convert(
|
@@ -35,6 +37,8 @@ module Stepmod
|
|
35
37
|
).strip
|
36
38
|
end
|
37
39
|
|
40
|
+
# TODO: `designations:` should include the `alt:[...]` terms here,
|
41
|
+
# they are now only included in definition_xml_converted_definition.
|
38
42
|
new(
|
39
43
|
designations: [designation],
|
40
44
|
definition: definition,
|
@@ -26,7 +26,8 @@ module Stepmod
|
|
26
26
|
:part_concepts,
|
27
27
|
:part_resources,
|
28
28
|
:part_modules,
|
29
|
-
:stdout
|
29
|
+
:stdout,
|
30
|
+
:git_rev
|
30
31
|
|
31
32
|
def self.call(stepmod_dir, index_path, stdout = $stdout)
|
32
33
|
new(stepmod_dir, index_path, stdout).call
|
@@ -65,6 +66,12 @@ module Stepmod
|
|
65
66
|
log "INFO: STEPmod directory set to #{stepmod_dir}."
|
66
67
|
log "INFO: Detecting paths..."
|
67
68
|
|
69
|
+
# Run `cvs status` to find out version
|
70
|
+
log "INFO: Detecting Git SHA..."
|
71
|
+
Dir.chdir(stepmod_path) do
|
72
|
+
@git_rev = `git rev-parse HEAD` || nil
|
73
|
+
end
|
74
|
+
|
68
75
|
repo_index = Nokogiri::XML(File.read(@index_path)).root
|
69
76
|
|
70
77
|
files = []
|
@@ -134,8 +141,8 @@ module Stepmod
|
|
134
141
|
end
|
135
142
|
|
136
143
|
unless ACCEPTED_STAGES.include? bibdata.doctype
|
137
|
-
log "INFO: skipped #{bibdata.docid} as it is not \
|
138
|
-
one of (#{ACCEPTED_STAGES.join(', ')})."
|
144
|
+
log "INFO: skipped #{bibdata.docid} as it is not " \
|
145
|
+
"one of (#{ACCEPTED_STAGES.join(', ')})."
|
139
146
|
next
|
140
147
|
end
|
141
148
|
|
@@ -145,23 +152,9 @@ module Stepmod
|
|
145
152
|
next
|
146
153
|
end
|
147
154
|
|
148
|
-
revision_string = ""
|
149
|
-
|
150
|
-
# Run `cvs status` to find out version
|
151
|
-
log "INFO: Detecting Git SHA..."
|
152
|
-
Dir.chdir(stepmod_path) do
|
153
|
-
git_sha = `git rev-parse HEAD`
|
154
|
-
|
155
|
-
unless git_sha.empty?
|
156
|
-
revision_string = "\n// Git: SHA #{git_sha}"
|
157
|
-
end
|
158
|
-
end
|
159
|
-
|
160
155
|
# read definitions
|
161
156
|
current_part_concepts = Glossarist::Collection.new
|
162
|
-
|
163
|
-
current_document.xpath("//definition").each do |definition|
|
164
|
-
definition_index += 1
|
157
|
+
current_document.xpath("//definition").each.with_index(1) do |definition, definition_index|
|
165
158
|
term_id = definition["id"]
|
166
159
|
unless term_id.nil?
|
167
160
|
if encountered_terms[term_id]
|
@@ -178,7 +171,7 @@ module Stepmod
|
|
178
171
|
definition,
|
179
172
|
reference_anchor: bibdata.anchor,
|
180
173
|
reference_clause: ref_clause,
|
181
|
-
file_path: fpath
|
174
|
+
file_path: fpath,
|
182
175
|
)
|
183
176
|
next unless concept
|
184
177
|
|
@@ -197,10 +190,10 @@ module Stepmod
|
|
197
190
|
current_part_modules_arm = {}
|
198
191
|
current_part_modules_mim = {}
|
199
192
|
|
200
|
-
log "INFO: FILE PATH IS #{file_path}"
|
193
|
+
# log "INFO: FILE PATH IS #{file_path}"
|
201
194
|
case file_path.to_s
|
202
195
|
when /resource.xml$/
|
203
|
-
log "INFO: Processing resource.xml for #{
|
196
|
+
log "INFO: Processing resource.xml for #{fpath}"
|
204
197
|
|
205
198
|
current_document.xpath("//schema").each do |schema_node|
|
206
199
|
schema_name = schema_node["name"]
|
@@ -259,7 +252,7 @@ module Stepmod
|
|
259
252
|
end
|
260
253
|
|
261
254
|
when /module.xml$/
|
262
|
-
log "INFO: Processing module.xml for #{
|
255
|
+
log "INFO: Processing module.xml for #{fpath}"
|
263
256
|
# Assumption: every schema is only linked by a single module document.
|
264
257
|
# puts current_document.xpath('//module').length
|
265
258
|
schema_name = current_document.xpath("//module").first["name"]
|
@@ -272,101 +265,36 @@ module Stepmod
|
|
272
265
|
parsed_schema_names[schema_name] = file_path
|
273
266
|
end
|
274
267
|
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
repo = Expressir::Express::Parser.from_file(exp_annotated_path)
|
282
|
-
|
283
|
-
repo.schemas.each do |schema|
|
284
|
-
schema.entities.each do |entity|
|
285
|
-
old_definition = entity.remarks.first
|
286
|
-
|
287
|
-
domain = "application module: #{schema.id}"
|
288
|
-
entity_definition = generate_entity_definition(entity, domain, old_definition)
|
289
|
-
|
290
|
-
reference_anchor = bibdata.anchor
|
291
|
-
reference_clause = nil
|
292
|
-
|
293
|
-
concept = Stepmod::Utils::Concept.new(
|
294
|
-
designations: [entity.id],
|
295
|
-
definition: old_definition,
|
296
|
-
converted_definition: entity_definition,
|
297
|
-
id: "#{reference_anchor}.#{reference_clause}",
|
298
|
-
reference_anchor: reference_anchor,
|
299
|
-
reference_clause: reference_clause,
|
300
|
-
file_path: Pathname.new(exp_annotated_path)
|
301
|
-
.relative_path_from(stepmod_path),
|
302
|
-
language_code: "en",
|
303
|
-
)
|
304
|
-
|
305
|
-
next unless concept
|
268
|
+
arm_schema, arm_concepts = parse_annotated_module(
|
269
|
+
type: :arm,
|
270
|
+
stepmod_path: stepmod_path,
|
271
|
+
path: "modules/#{schema_name}/arm_annotated.exp",
|
272
|
+
bibdata: bibdata
|
273
|
+
)
|
306
274
|
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
275
|
+
mim_schema, mim_concepts = parse_annotated_module(
|
276
|
+
type: :mim,
|
277
|
+
stepmod_path: stepmod_path,
|
278
|
+
path: "modules/#{schema_name}/mim_annotated.exp",
|
279
|
+
bibdata: bibdata
|
280
|
+
)
|
312
281
|
|
313
|
-
|
314
|
-
|
315
|
-
end
|
316
|
-
end
|
282
|
+
if arm_concepts.to_a.size > 0
|
283
|
+
current_part_modules_arm[arm_schema] = arm_concepts
|
317
284
|
end
|
318
285
|
|
319
|
-
|
320
|
-
|
321
|
-
log "INFO: Processing modules schema #{mim_exp_annotated_path}"
|
322
|
-
|
323
|
-
if File.exists?(mim_exp_annotated_path)
|
324
|
-
repo = Expressir::Express::Parser.from_file(mim_exp_annotated_path)
|
325
|
-
|
326
|
-
repo.schemas.each do |schema|
|
327
|
-
schema.entities.each do |entity|
|
328
|
-
old_definition = entity.remarks.first
|
329
|
-
|
330
|
-
domain = "application module: #{schema.id}"
|
331
|
-
definition = generate_entity_definition(entity, domain, old_definition)
|
332
|
-
|
333
|
-
reference_anchor = bibdata.anchor
|
334
|
-
reference_clause = nil
|
335
|
-
|
336
|
-
concept = Stepmod::Utils::Concept.new(
|
337
|
-
designations: [entity.id],
|
338
|
-
definition: old_definition,
|
339
|
-
converted_definition: definition,
|
340
|
-
id: "#{reference_anchor}.#{reference_clause}",
|
341
|
-
reference_anchor: reference_anchor,
|
342
|
-
reference_clause: reference_clause,
|
343
|
-
file_path: Pathname.new(exp_annotated_path)
|
344
|
-
.relative_path_from(stepmod_path),
|
345
|
-
language_code: "en",
|
346
|
-
)
|
347
|
-
|
348
|
-
next unless concept
|
349
|
-
|
350
|
-
current_part_modules_mim[schema.id] ||=
|
351
|
-
Glossarist::Collection.new
|
352
|
-
find_or_initialize_concept(
|
353
|
-
current_part_modules_mim[schema.id], concept
|
354
|
-
)
|
355
|
-
|
356
|
-
parsed_bibliography << bibdata
|
357
|
-
end
|
358
|
-
end
|
286
|
+
if mim_concepts.to_a.size > 0
|
287
|
+
current_part_modules_mim[mim_schema] = mim_concepts
|
359
288
|
end
|
360
|
-
|
361
289
|
end
|
362
290
|
|
363
291
|
log "INFO: Completed processing XML file #{fpath}"
|
364
292
|
if current_part_concepts.to_a.empty?
|
365
|
-
log "INFO: Skipping #{fpath} (#{bibdata.docid}) \
|
366
|
-
because it contains no concepts."
|
293
|
+
log "INFO: Skipping #{fpath} (#{bibdata.docid}) " \
|
294
|
+
"because it contains no concepts."
|
367
295
|
elsif current_part_concepts.to_a.length < 3
|
368
|
-
log "INFO: Skipping #{fpath} (#{bibdata.docid}) \
|
369
|
-
because it only has #{current_part_concepts.to_a.length} terms."
|
296
|
+
log "INFO: Skipping #{fpath} (#{bibdata.docid}) " \
|
297
|
+
"because it only has #{current_part_concepts.to_a.length} terms."
|
370
298
|
|
371
299
|
current_part_concepts.to_a.each do |x|
|
372
300
|
general_concepts.store(x)
|
@@ -377,23 +305,85 @@ module Stepmod
|
|
377
305
|
current_part_concepts]
|
378
306
|
end
|
379
307
|
end
|
308
|
+
|
380
309
|
unless current_part_resources.to_a.empty?
|
381
310
|
part_resources << [bibdata,
|
382
311
|
current_part_resources]
|
383
312
|
end
|
313
|
+
|
384
314
|
if (current_part_modules_arm.to_a.size +
|
385
315
|
current_part_modules_mim.to_a.size).positive?
|
316
|
+
|
386
317
|
part_modules << [bibdata, current_part_modules_arm,
|
387
318
|
current_part_modules_mim]
|
319
|
+
parsed_bibliography << bibdata
|
388
320
|
end
|
321
|
+
|
389
322
|
end
|
390
323
|
end
|
391
324
|
|
392
|
-
def
|
393
|
-
|
325
|
+
def parse_annotated_module(type:, stepmod_path:, path:, bibdata:)
|
326
|
+
log "INFO: parse_annotated_module: Processing modules schema #{path}"
|
327
|
+
|
328
|
+
fpath = File.join(stepmod_path, path)
|
329
|
+
|
330
|
+
unless File.exists?(fpath)
|
331
|
+
log "ERROR: parse_annotated_module: No module schema exists at #{fpath}."
|
332
|
+
return
|
333
|
+
end
|
334
|
+
|
335
|
+
repo = Expressir::Express::Parser.from_file(fpath)
|
336
|
+
|
337
|
+
unless repo
|
338
|
+
log "ERROR: parse_annotated_module: failed to parse EXPRESS file at #{path}."
|
339
|
+
return
|
340
|
+
end
|
341
|
+
|
342
|
+
# See: metanorma/iso-10303-2#90
|
343
|
+
domain_prefix = case type
|
344
|
+
when :mim
|
345
|
+
"application module"
|
346
|
+
when :arm
|
347
|
+
"application object"
|
348
|
+
end
|
394
349
|
|
350
|
+
if repo.schemas.length > 1
|
351
|
+
raise StandardError.new(
|
352
|
+
"ERROR: FATAL: #{fpath} contains more than one schema:" +
|
353
|
+
"#{repo.schemas.map(&:id).join(", ")} (not supposed to happen!!)"
|
354
|
+
)
|
355
|
+
end
|
356
|
+
|
357
|
+
schema = repo.schemas.first
|
358
|
+
collection = Glossarist::Collection.new
|
359
|
+
domain = "#{domain_prefix}: #{schema.id}"
|
360
|
+
|
361
|
+
schema.entities.each do |entity|
|
362
|
+
old_definition = entity.remarks.first
|
363
|
+
new_definition = generate_entity_definition(entity, domain, old_definition)
|
364
|
+
|
365
|
+
concept = Stepmod::Utils::Concept.new(
|
366
|
+
designations: [entity.id],
|
367
|
+
definition: old_definition,
|
368
|
+
converted_definition: new_definition,
|
369
|
+
# TODO: Find a proper ID for this
|
370
|
+
id: "#{bibdata.anchor}.",
|
371
|
+
reference_anchor: bibdata.anchor,
|
372
|
+
reference_clause: nil,
|
373
|
+
file_path: path,
|
374
|
+
language_code: "en",
|
375
|
+
)
|
376
|
+
|
377
|
+
next unless concept
|
378
|
+
find_or_initialize_concept(collection, concept)
|
379
|
+
end
|
380
|
+
|
381
|
+
[schema.id, collection]
|
382
|
+
end
|
383
|
+
|
384
|
+
def find_or_initialize_concept(collection, localized_concept)
|
395
385
|
concept = collection
|
396
|
-
.store(Glossarist::Concept.new(id:
|
386
|
+
.store(Glossarist::Concept.new(id: SecureRandom.uuid))
|
397
387
|
concept.add_l10n(localized_concept)
|
398
388
|
end
|
399
389
|
|
@@ -454,7 +444,9 @@ module Stepmod
|
|
454
444
|
combined.gsub!(/\n\/\/.*?\n/, "\n")
|
455
445
|
combined.strip!
|
456
446
|
|
457
|
-
combined
|
447
|
+
express_reference_to_mention(combined)
|
448
|
+
|
449
|
+
# combined
|
458
450
|
# # TODO: If the definition contains a list immediately after the first paragraph, don't split
|
459
451
|
# return definition if definition =~ /\n\* /
|
460
452
|
|
@@ -471,19 +463,58 @@ module Stepmod
|
|
471
463
|
# end
|
472
464
|
end
|
473
465
|
|
466
|
+
# Replace `<<express:{schema}.{entity},{render}>>` with {{entity,render}}
|
467
|
+
def express_reference_to_mention(description)
|
468
|
+
# TODO: Use Expressir to check whether the "entity" is really an
|
469
|
+
# EXPRESS ENTITY. If not, skip the mention.
|
470
|
+
description.gsub(/<<express:([^,]+),([^>]+)>>/) do |match|
|
471
|
+
"{{#{Regexp.last_match[1].split('.').last},#{Regexp.last_match[2]}}}"
|
472
|
+
end
|
473
|
+
end
|
474
|
+
|
475
|
+
def entity_name_to_text(entity_id)
|
476
|
+
entity_id.downcase.gsub(/_/, " ")
|
477
|
+
end
|
474
478
|
|
475
479
|
# rubocop:disable Layout/LineLength
|
476
480
|
def generate_entity_definition(entity, domain, old_definition)
|
477
481
|
return "" if entity.nil?
|
478
482
|
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
|
486
|
-
|
483
|
+
# See: metanorma/iso-10303-2#90
|
484
|
+
# TODO: This is not DRY in case we have to further customize
|
485
|
+
entity_text = if domain_type = domain.match(/\A(application object):/)
|
486
|
+
|
487
|
+
entity_ref = if entity.id == entity_name_to_text(entity.id)
|
488
|
+
"{{#{entity.id}}}"
|
489
|
+
else
|
490
|
+
"{{#{entity.id},#{entity_name_to_text(entity.id)}}}"
|
491
|
+
end
|
492
|
+
|
493
|
+
if entity.subtype_of.size.zero?
|
494
|
+
"#{domain_type[1]} that represents the " + entity_ref + " entity"
|
495
|
+
else
|
496
|
+
entity_subtypes = entity.subtype_of.map do |e|
|
497
|
+
"{{#{e.id},#{entity_name_to_text(e.id)}}}"
|
498
|
+
end
|
499
|
+
"#{domain_type[1]} that is a type of " +
|
500
|
+
"#{entity_subtypes.join(' and ')} that represents the " +
|
501
|
+
entity_ref + " entity"
|
502
|
+
end
|
503
|
+
|
504
|
+
else
|
505
|
+
|
506
|
+
# Not "application object"
|
507
|
+
if entity.subtype_of.size.zero?
|
508
|
+
"entity data type that represents the {{#{entity.id}}} entity"
|
509
|
+
else
|
510
|
+
entity_subtypes = entity.subtype_of.map do |e|
|
511
|
+
"{{#{e.id}}}"
|
512
|
+
end
|
513
|
+
"entity data type that is a type of " +
|
514
|
+
"#{entity_subtypes.join(' and ')} that represents the " +
|
515
|
+
"{{#{entity.id}}} entity"
|
516
|
+
end
|
517
|
+
end
|
487
518
|
|
488
519
|
definition = <<~DEFINITION
|
489
520
|
=== #{entity.id}
|
@@ -493,6 +524,7 @@ module Stepmod
|
|
493
524
|
|
494
525
|
DEFINITION
|
495
526
|
|
527
|
+
# If there is a definition, we add it as the first NOTE
|
496
528
|
unless old_definition.nil? || old_definition.blank?
|
497
529
|
old_definition = trim_definition(old_definition)
|
498
530
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: stepmod-utils
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.21
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ribose Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-09-
|
11
|
+
date: 2022-09-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|