stepmod-utils 0.4.13 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/release.yml +4 -3
- data/exe/stepmod-annotate-all +5 -4
- data/exe/stepmod-convert-express-description +4 -6
- data/exe/stepmod-convert-express-resource +4 -6
- data/exe/stepmod-extract-concepts +5 -4
- data/exe/stepmod-srl-migrate +53 -21
- data/exe/stepmod-validate-express +13 -15
- data/lib/stepmod/utils/change.rb +47 -4
- data/lib/stepmod/utils/change_collection.rb +1 -1
- data/lib/stepmod/utils/change_edition.rb +6 -6
- data/lib/stepmod/utils/changes_extractor.rb +46 -10
- data/lib/stepmod/utils/concept.rb +12 -10
- data/lib/stepmod/utils/converters/blockquote.rb +0 -2
- data/lib/stepmod/utils/converters/def.rb +4 -4
- data/lib/stepmod/utils/converters/dt.rb +1 -1
- data/lib/stepmod/utils/converters/em_express_description.rb +8 -7
- data/lib/stepmod/utils/converters/eqn.rb +8 -6
- data/lib/stepmod/utils/converters/express_figure.rb +1 -1
- data/lib/stepmod/utils/converters/express_g.rb +1 -1
- data/lib/stepmod/utils/converters/express_table.rb +1 -1
- data/lib/stepmod/utils/converters/ext_description.rb +2 -1
- data/lib/stepmod/utils/converters/figure.rb +5 -5
- data/lib/stepmod/utils/converters/module_ref.rb +3 -5
- data/lib/stepmod/utils/converters/module_ref_express_description.rb +3 -7
- data/lib/stepmod/utils/converters/ol.rb +7 -7
- data/lib/stepmod/utils/converters/stepmod_ext_description.rb +3 -3
- data/lib/stepmod/utils/converters/strong.rb +12 -11
- data/lib/stepmod/utils/converters/term.rb +0 -1
- data/lib/stepmod/utils/express_bibdata.rb +6 -2
- data/lib/stepmod/utils/html_to_asciimath.rb +80 -80
- data/lib/stepmod/utils/parsers/mapping_table_parser.rb +31 -0
- data/lib/stepmod/utils/parsers/models/a.rb +40 -0
- data/lib/stepmod/utils/parsers/models/aa.rb +53 -0
- data/lib/stepmod/utils/parsers/models/ae.rb +54 -0
- data/lib/stepmod/utils/parsers/models/aimelt.rb +21 -0
- data/lib/stepmod/utils/parsers/models/alt.rb +19 -0
- data/lib/stepmod/utils/parsers/models/alt_map.rb +46 -0
- data/lib/stepmod/utils/parsers/models/alt_scmap.rb +34 -0
- data/lib/stepmod/utils/parsers/models/b.rb +58 -0
- data/lib/stepmod/utils/parsers/models/bigeqn.rb +55 -0
- data/lib/stepmod/utils/parsers/models/bold.rb +52 -0
- data/lib/stepmod/utils/parsers/models/bom_ref.rb +21 -0
- data/lib/stepmod/utils/parsers/models/constraint.rb +19 -0
- data/lib/stepmod/utils/parsers/models/dd.rb +86 -0
- data/lib/stepmod/utils/parsers/models/description.rb +86 -0
- data/lib/stepmod/utils/parsers/models/dl.rb +24 -0
- data/lib/stepmod/utils/parsers/models/dt.rb +86 -0
- data/lib/stepmod/utils/parsers/models/eqn.rb +57 -0
- data/lib/stepmod/utils/parsers/models/example.rb +84 -0
- data/lib/stepmod/utils/parsers/models/express_extref.rb +21 -0
- data/lib/stepmod/utils/parsers/models/express_ref.rb +21 -0
- data/lib/stepmod/utils/parsers/models/figure.rb +30 -0
- data/lib/stepmod/utils/parsers/models/i.rb +52 -0
- data/lib/stepmod/utils/parsers/models/img.rb +25 -0
- data/lib/stepmod/utils/parsers/models/imgarea.rb +23 -0
- data/lib/stepmod/utils/parsers/models/li.rb +86 -0
- data/lib/stepmod/utils/parsers/models/mapping_table.rb +26 -0
- data/lib/stepmod/utils/parsers/models/module_ref.rb +21 -0
- data/lib/stepmod/utils/parsers/models/note.rb +84 -0
- data/lib/stepmod/utils/parsers/models/ol.rb +25 -0
- data/lib/stepmod/utils/parsers/models/p.rb +56 -0
- data/lib/stepmod/utils/parsers/models/refpath.rb +23 -0
- data/lib/stepmod/utils/parsers/models/refpath_extend.rb +23 -0
- data/lib/stepmod/utils/parsers/models/rules.rb +19 -0
- data/lib/stepmod/utils/parsers/models/sc.rb +36 -0
- data/lib/stepmod/utils/parsers/models/screen.rb +55 -0
- data/lib/stepmod/utils/parsers/models/source.rb +21 -0
- data/lib/stepmod/utils/parsers/models/sub.rb +52 -0
- data/lib/stepmod/utils/parsers/models/sup.rb +52 -0
- data/lib/stepmod/utils/parsers/models/table.rb +29 -0
- data/lib/stepmod/utils/parsers/models/td.rb +98 -0
- data/lib/stepmod/utils/parsers/models/th.rb +98 -0
- data/lib/stepmod/utils/parsers/models/title.rb +53 -0
- data/lib/stepmod/utils/parsers/models/tr.rb +26 -0
- data/lib/stepmod/utils/parsers/models/tt.rb +68 -0
- data/lib/stepmod/utils/parsers/models/ul.rb +21 -0
- data/lib/stepmod/utils/parsers/models.rb +0 -0
- data/lib/stepmod/utils/parsers/types/string_without_indent.rb +17 -0
- data/lib/stepmod/utils/parsers.rb +4 -0
- data/lib/stepmod/utils/smrl_description_converter.rb +2 -1
- data/lib/stepmod/utils/smrl_resource_converter.rb +2 -1
- data/lib/stepmod/utils/stepmod_file_annotator.rb +14 -7
- data/lib/stepmod/utils/term.rb +1 -1
- data/lib/stepmod/utils/terms_extractor.rb +33 -31
- data/lib/stepmod/utils/version.rb +1 -1
- data/stepmod-utils.gemspec +4 -3
- metadata +76 -13
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 18bfb358046e62d1504ff514004e1c01213a5cd3a86c53985dabcf5ede14e56c
|
|
4
|
+
data.tar.gz: 37bf06f7198d4066bbabba3038ab4107dd4ac16dc65add33d0cd267b26fd2bdd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 767a9ba90526a414738cbc5bafe61b0e80da0f5b7ed9a714aa10442f87ca200303bd565ad718b0e5c2f5255311d174400fa00be6932f2b291c0b1ee55ed597b8
|
|
7
|
+
data.tar.gz: a73c95f1f931e1023f44bf308888a38cb1f1f7bc570e7fe910131e50e843b6c2d0384bb863f33fe35456ffdd39512ca85bb0e85f13cb62c7b663dc7178d78e39
|
|
@@ -7,11 +7,12 @@ on:
|
|
|
7
7
|
inputs:
|
|
8
8
|
next_version:
|
|
9
9
|
description: |
|
|
10
|
-
Next release version. Possible values: x.y.z, major, minor, patch or pre|rc|etc
|
|
10
|
+
Next release version. Possible values: x.y.z, major, minor, patch (or pre|rc|etc).
|
|
11
|
+
Also, you can pass 'skip' to skip 'git tag' and do 'gem push' for the current version
|
|
11
12
|
required: true
|
|
12
13
|
default: 'skip'
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
repository_dispatch:
|
|
15
|
+
types: [ do-release ]
|
|
15
16
|
|
|
16
17
|
jobs:
|
|
17
18
|
release:
|
data/exe/stepmod-annotate-all
CHANGED
|
@@ -58,17 +58,20 @@ stepmod_dir = options[:stepmod_dir]
|
|
|
58
58
|
raise StandardError.new("--stepmod-dir not set") unless stepmod_dir
|
|
59
59
|
|
|
60
60
|
stepmod_path = Pathname.new(stepmod_dir)
|
|
61
|
-
unless
|
|
61
|
+
unless stepmod_path.join("data").exist? && stepmod_path.join("data").directory?
|
|
62
62
|
raise StandardError.new("--stepmod-dir should be the root STEPmod directory and contains data/")
|
|
63
63
|
end
|
|
64
|
+
|
|
64
65
|
log "STEPmod data path: `#{stepmod_dir}`"
|
|
65
66
|
|
|
66
67
|
srl_output_schemas_dir = options[:srl_output_schemas_dir]
|
|
67
68
|
raise StandardError.new("--schemas directory for SRL schemas not set") unless srl_output_schemas_dir
|
|
69
|
+
|
|
68
70
|
log "SRL schemas output path: `#{srl_output_schemas_dir}`"
|
|
69
71
|
|
|
70
72
|
srl_output_docs_dir = options[:srl_output_docs_dir]
|
|
71
73
|
raise StandardError.new("--documents directory for SRL documents (generated by stepmod2mn) not set") unless srl_output_docs_dir
|
|
74
|
+
|
|
72
75
|
log "SRL documents path: `#{srl_output_docs_dir}`"
|
|
73
76
|
|
|
74
77
|
unless File.exist?(srl_output_docs_dir) && File.directory?(srl_output_docs_dir)
|
|
@@ -126,7 +129,7 @@ def file_system_path(filepath, filename_cache)
|
|
|
126
129
|
filename_cache[dir][filepath.downcase]
|
|
127
130
|
end
|
|
128
131
|
|
|
129
|
-
MAX_THREADS = 1 #[2, Concurrent.processor_count].max * 2
|
|
132
|
+
MAX_THREADS = 1 # [2, Concurrent.processor_count].max * 2
|
|
130
133
|
MAX_QUEUE_SIZE = MAX_THREADS * 4
|
|
131
134
|
# https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/thread_pools.md
|
|
132
135
|
# pool = Concurrent::ThreadPoolExecutor.new(
|
|
@@ -143,7 +146,6 @@ FileUtils.mkdir_p(srl_output_schemas_dir)
|
|
|
143
146
|
srl_output_schemas_path = Pathname.new(srl_output_schemas_dir)
|
|
144
147
|
srl_output_docs_path = Pathname.new(srl_output_docs_dir)
|
|
145
148
|
|
|
146
|
-
|
|
147
149
|
files.each do |file|
|
|
148
150
|
puts "#{Thread.current.object_id}: `#{file}` processing..."
|
|
149
151
|
|
|
@@ -153,7 +155,6 @@ files.each do |file|
|
|
|
153
155
|
).call
|
|
154
156
|
|
|
155
157
|
destination_rel_path = Pathname.new(file).dirname.relative_path_from(stepmod_path.join("data"))
|
|
156
|
-
directory_type = destination_rel_path.each_filename.to_a.first
|
|
157
158
|
|
|
158
159
|
destination_path = srl_output_schemas_path.join(destination_rel_path)
|
|
159
160
|
FileUtils.mkdir_p(destination_path)
|
|
@@ -17,10 +17,8 @@ input_content = ARGF.read
|
|
|
17
17
|
|
|
18
18
|
result = Stepmod::Utils::SmrlDescriptionConverter.convert(input_content, {})
|
|
19
19
|
result.split("\n").each do |line|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
exit(74)
|
|
25
|
-
end
|
|
20
|
+
$stdout.puts(line)
|
|
21
|
+
$stdout.flush
|
|
22
|
+
rescue Errno::EPIPE
|
|
23
|
+
exit(74)
|
|
26
24
|
end
|
|
@@ -17,10 +17,8 @@ input_content = ARGF.read
|
|
|
17
17
|
|
|
18
18
|
result = Stepmod::Utils::SmrlResourceConverter.convert(input_content, {})
|
|
19
19
|
result.split("\n").each do |line|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
exit(74)
|
|
25
|
-
end
|
|
20
|
+
$stdout.puts(line)
|
|
21
|
+
$stdout.flush
|
|
22
|
+
rescue Errno::EPIPE
|
|
23
|
+
exit(74)
|
|
26
24
|
end
|
|
@@ -77,7 +77,8 @@ else
|
|
|
77
77
|
)
|
|
78
78
|
end
|
|
79
79
|
|
|
80
|
-
default_output_dir = File.join(File.expand_path("..", stepmod_dir),
|
|
80
|
+
default_output_dir = File.join(File.expand_path("..", stepmod_dir),
|
|
81
|
+
"output_yaml")
|
|
81
82
|
output_dir = options[:output_dir] || default_output_dir
|
|
82
83
|
unless File.directory?(output_dir)
|
|
83
84
|
FileUtils.mkdir_p(output_dir)
|
|
@@ -164,7 +165,7 @@ end
|
|
|
164
165
|
yaml_outputs = {
|
|
165
166
|
clause_4_sections: {},
|
|
166
167
|
clause_5_sections: {},
|
|
167
|
-
bibliographies: {}
|
|
168
|
+
bibliographies: {},
|
|
168
169
|
}
|
|
169
170
|
|
|
170
171
|
part_resources.each do |(_bibdata, current_part_resources)|
|
|
@@ -204,9 +205,9 @@ extract_bibliographies(resource_concepts, yaml_outputs[:bibliographies])
|
|
|
204
205
|
{
|
|
205
206
|
clause_4_sections: "resource_sections.yaml",
|
|
206
207
|
clause_5_sections: "module_sections.yaml",
|
|
207
|
-
bibliographies: "bib.yaml"
|
|
208
|
+
bibliographies: "bib.yaml",
|
|
208
209
|
}.each_pair do |var, filename|
|
|
209
210
|
path = File.join(output_dir, filename)
|
|
210
211
|
File.write(path, yaml_outputs[var].values.to_yaml)
|
|
211
|
-
log "INFO: #{var
|
|
212
|
+
log "INFO: #{var} written to #{path}."
|
|
212
213
|
end
|
data/exe/stepmod-srl-migrate
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
require "pathname"
|
|
6
6
|
require "fileutils"
|
|
7
|
+
require "nokogiri"
|
|
7
8
|
bin_file = Pathname.new(__FILE__).realpath
|
|
8
9
|
|
|
9
10
|
# add self to libpath
|
|
@@ -14,14 +15,14 @@ require "optparse"
|
|
|
14
15
|
|
|
15
16
|
def log(message, indent = 0)
|
|
16
17
|
indent_spaces = " " * (indent * 2)
|
|
17
|
-
puts "[stepmod-utils] #{message}"
|
|
18
|
+
puts "[stepmod-utils] #{indent_spaces}#{message}"
|
|
18
19
|
end
|
|
19
20
|
|
|
20
|
-
def copy_files_to_schemas(path, stepmod_path,
|
|
21
|
+
def copy_files_to_schemas(path, stepmod_path, srl_output_schemas_dir)
|
|
21
22
|
base_path = path.dirname.relative_path_from(File.join(stepmod_path, "data"))
|
|
22
23
|
file_name = path.basename
|
|
23
24
|
|
|
24
|
-
new_dir = File.join(
|
|
25
|
+
new_dir = File.join(srl_output_schemas_dir, base_path.to_s)
|
|
25
26
|
new_file_path = File.join(new_dir, file_name)
|
|
26
27
|
|
|
27
28
|
unless Dir.exist?(new_dir)
|
|
@@ -32,6 +33,28 @@ def copy_files_to_schemas(path, stepmod_path, srl_output_docs_dir)
|
|
|
32
33
|
log("Copied #{path.basename} to #{new_file_path}", 1)
|
|
33
34
|
end
|
|
34
35
|
|
|
36
|
+
def copy_files_to_document(path, stepmod_path, srl_output_docs_dir)
|
|
37
|
+
file_name = path.basename
|
|
38
|
+
module_xml_path = path.dirname.join("module.xml")
|
|
39
|
+
# puts "module_xml_path #{module_xml_path}"
|
|
40
|
+
module_xml = Nokogiri::XML(IO.read(module_xml_path)).root
|
|
41
|
+
|
|
42
|
+
part_number = module_xml.xpath("//module").first.attr("part")
|
|
43
|
+
# module_name = module_xml.xpath('//module').first.attr('name')
|
|
44
|
+
|
|
45
|
+
new_dir = File.join(srl_output_docs_dir, "iso-10303-#{part_number}")
|
|
46
|
+
new_file_path = File.join(new_dir, file_name)
|
|
47
|
+
# new_file_path = File.join(new_dir, "#{module_name}.#{file_name}")
|
|
48
|
+
|
|
49
|
+
## The document directory should already exist.
|
|
50
|
+
# unless Dir.exist?(new_dir)
|
|
51
|
+
# FileUtils.mkdir_p(new_dir)
|
|
52
|
+
# end
|
|
53
|
+
|
|
54
|
+
FileUtils.copy_file(path, new_file_path)
|
|
55
|
+
log("Copied #{path.basename} to #{new_file_path}", 1)
|
|
56
|
+
end
|
|
57
|
+
|
|
35
58
|
options = {}
|
|
36
59
|
OptionParser.new do |opts|
|
|
37
60
|
opts.banner = "Usage: #{$0} [options]"
|
|
@@ -75,19 +98,19 @@ OptionParser.new do |opts|
|
|
|
75
98
|
end.parse!
|
|
76
99
|
|
|
77
100
|
def download_stepmod2mn(path)
|
|
78
|
-
if path
|
|
101
|
+
if path&.exist?
|
|
79
102
|
puts "Specified `stepmod2mn.jar` found, skipping download."
|
|
80
103
|
return path
|
|
81
104
|
end
|
|
82
105
|
|
|
83
106
|
puts "Specified `stepmod2mn.jar` not found, downloading from GitHub."
|
|
84
107
|
|
|
85
|
-
require
|
|
108
|
+
require "octokit"
|
|
86
109
|
require "down"
|
|
87
110
|
require "fileutils"
|
|
88
111
|
|
|
89
112
|
client = Octokit::Client.new
|
|
90
|
-
download_url = client.latest_release(
|
|
113
|
+
download_url = client.latest_release("metanorma/stepmod2mn").assets.first.browser_download_url
|
|
91
114
|
|
|
92
115
|
filename = Pathname.new(download_url).basename.to_s
|
|
93
116
|
|
|
@@ -123,7 +146,7 @@ end
|
|
|
123
146
|
|
|
124
147
|
stepmod_path = Pathname.new(stepmod_dir)
|
|
125
148
|
unless stepmod_path.exist? && stepmod_path.directory? &&
|
|
126
|
-
|
|
149
|
+
stepmod_path.join("data").exist? && stepmod_path.join("data").directory?
|
|
127
150
|
puts "Specified STEPmod directory not found, not a directory, or does not contain `data/`. Aborting."
|
|
128
151
|
exit 1
|
|
129
152
|
end
|
|
@@ -143,42 +166,51 @@ unless File.directory?(srl_output_docs_dir)
|
|
|
143
166
|
end
|
|
144
167
|
log "SRL output documents directory path: `#{srl_output_docs_dir}`"
|
|
145
168
|
|
|
146
|
-
log("*"*30)
|
|
169
|
+
log("*" * 30)
|
|
147
170
|
log("[SRL MIGRATION] Migration starts!")
|
|
148
171
|
|
|
149
|
-
log("*"*30)
|
|
172
|
+
log("*" * 30)
|
|
150
173
|
log("[SRL MIGRATION: stepmod2mn.jar] START SVG generation.")
|
|
151
|
-
log("*"*30)
|
|
174
|
+
log("*" * 30)
|
|
152
175
|
system "java -Xss5m -jar #{stepmod2mn_path} #{stepmod_dir} --output-documents #{srl_output_docs_dir} --output-schemas #{srl_output_schemas_dir} --svg"
|
|
153
176
|
|
|
154
|
-
log("*"*30)
|
|
177
|
+
log("*" * 30)
|
|
155
178
|
log("[SRL MIGRATION: stepmod2mn.jar] COMPLETE SVG generation.")
|
|
156
179
|
|
|
157
|
-
log("*"*30)
|
|
180
|
+
log("*" * 30)
|
|
158
181
|
log("[SRL MIGRATION: stepmod2mn.jar] START document migration and SVG generation.")
|
|
159
|
-
log("*"*30)
|
|
182
|
+
log("*" * 30)
|
|
160
183
|
system "java -Xss5m -jar #{stepmod2mn_path} #{stepmod_dir} --output-documents #{srl_output_docs_dir} --output-schemas #{srl_output_schemas_dir}"
|
|
161
|
-
log("*"*30)
|
|
184
|
+
log("*" * 30)
|
|
162
185
|
log("[SRL MIGRATION: stepmod2mn.jar] COMPLETE document migration and SVG generation.")
|
|
163
186
|
|
|
164
|
-
log("*"*30)
|
|
187
|
+
log("*" * 30)
|
|
165
188
|
log("[SRL MIGRATION: stepmod-utils] START Annotated EXPRESS generation.")
|
|
166
|
-
log("*"*30)
|
|
189
|
+
log("*" * 30)
|
|
167
190
|
system "stepmod-annotate-all --stepmod-dir #{stepmod_dir} --documents #{srl_output_docs_dir} --schemas #{srl_output_schemas_dir}"
|
|
168
|
-
log("*"*30)
|
|
191
|
+
log("*" * 30)
|
|
169
192
|
log("[SRL MIGRATION: stepmod-utils] COMPLETE Annotated EXPRESS generation.")
|
|
170
193
|
|
|
171
|
-
log("*"*30)
|
|
194
|
+
log("*" * 30)
|
|
172
195
|
log("[SRL MIGRATION: stepmod-utils] START EXPRESS change YAML extraction.")
|
|
173
196
|
system "stepmod-extract-changes -p #{stepmod_dir}"
|
|
197
|
+
|
|
198
|
+
# Move arm/mim/arm_lf/mim_lf/mapping changes into schema directories
|
|
174
199
|
Dir.glob("#{stepmod_dir}/data/**/*.changes.yaml").each do |filepath|
|
|
175
200
|
path = Pathname.new(filepath)
|
|
176
201
|
copy_files_to_schemas(path, stepmod_dir, srl_output_schemas_dir)
|
|
177
202
|
end
|
|
178
|
-
|
|
203
|
+
|
|
204
|
+
# Move document changes into document directories
|
|
205
|
+
Dir.glob("#{stepmod_dir}/data/**/changes.yaml").each do |filepath|
|
|
206
|
+
path = Pathname.new(filepath)
|
|
207
|
+
copy_files_to_document(path, stepmod_dir, srl_output_docs_dir)
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
log("*" * 30)
|
|
179
211
|
log("[SRL MIGRATION: stepmod-utils] COMPLETE EXPRESS change YAML extraction.")
|
|
180
212
|
|
|
181
|
-
log("*"*30)
|
|
213
|
+
log("*" * 30)
|
|
182
214
|
log("[SRL MIGRATION] Migration complete!")
|
|
183
215
|
|
|
184
|
-
log("*"*30)
|
|
216
|
+
log("*" * 30)
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
2
|
|
|
3
|
-
require
|
|
4
|
-
require
|
|
5
|
-
require
|
|
6
|
-
require
|
|
3
|
+
require "nokogiri"
|
|
4
|
+
require "expressir"
|
|
5
|
+
require "expressir/express/parser"
|
|
6
|
+
require "expressir/express/formatter"
|
|
7
7
|
|
|
8
8
|
stepmod_dir = ARGV.pop || "iso-10303-stepmod-wg12"
|
|
9
9
|
stepmod_path = Pathname.new(stepmod_dir)
|
|
@@ -38,7 +38,7 @@ end
|
|
|
38
38
|
|
|
39
39
|
def parse_express_schema(path)
|
|
40
40
|
Expressir::Express::Parser.from_file(path)
|
|
41
|
-
rescue
|
|
41
|
+
rescue StandardError
|
|
42
42
|
puts "ERROR: Broken schema detected in #{path}!"
|
|
43
43
|
nil
|
|
44
44
|
end
|
|
@@ -54,7 +54,7 @@ puts REPO_INDEX_XML
|
|
|
54
54
|
puts
|
|
55
55
|
puts "NOTE: SHOULD_REWRITE mode is turned #{SHOULD_REWRITE ? 'ON' : 'OFF'}"
|
|
56
56
|
|
|
57
|
-
all_express_files(stepmod_dir).each_with_index do |exp_file,
|
|
57
|
+
all_express_files(stepmod_dir).each_with_index do |exp_file, _i|
|
|
58
58
|
# puts "i #{i}"
|
|
59
59
|
# next if i > 10
|
|
60
60
|
|
|
@@ -62,7 +62,6 @@ all_express_files(stepmod_dir).each_with_index do |exp_file, i|
|
|
|
62
62
|
exp_path = Pathname.new(exp_file)
|
|
63
63
|
repo = parse_express_schema(exp_file)
|
|
64
64
|
|
|
65
|
-
|
|
66
65
|
schemas_status[exp_path.to_s] = repo ? true : false
|
|
67
66
|
|
|
68
67
|
if repo.nil?
|
|
@@ -77,28 +76,27 @@ all_express_files(stepmod_dir).each_with_index do |exp_file, i|
|
|
|
77
76
|
formatted_schema = format_express_schema(repo)
|
|
78
77
|
# experimental_path = exp_path.dirname.join(exp_path.basename('.exp').to_s+".x.exp")
|
|
79
78
|
|
|
80
|
-
puts "rewriting : #{exp_path
|
|
79
|
+
puts "rewriting : #{exp_path}"
|
|
81
80
|
exp_path.truncate(0)
|
|
82
81
|
exp_path.write(formatted_schema)
|
|
83
82
|
end
|
|
84
|
-
|
|
85
83
|
end
|
|
86
84
|
|
|
87
|
-
puts "*"*30
|
|
85
|
+
puts "*" * 30
|
|
88
86
|
|
|
89
|
-
if schemas_status.
|
|
87
|
+
if schemas_status.value?(false)
|
|
90
88
|
puts "The following schemas failed validation:"
|
|
91
|
-
schemas_status.select do |
|
|
89
|
+
schemas_status.select do |_path, status|
|
|
92
90
|
status == false
|
|
93
|
-
end.each_pair do |path,
|
|
91
|
+
end.each_pair do |path, _status|
|
|
94
92
|
puts "SCHEMA: #{path}"
|
|
95
93
|
end
|
|
96
|
-
puts "*"*30
|
|
94
|
+
puts "*" * 30
|
|
97
95
|
|
|
98
96
|
exit 1
|
|
99
97
|
end
|
|
100
98
|
|
|
101
99
|
puts "All schemas passed validation."
|
|
102
|
-
puts "*"*30
|
|
100
|
+
puts "*" * 30
|
|
103
101
|
|
|
104
102
|
exit 0
|
data/lib/stepmod/utils/change.rb
CHANGED
|
@@ -5,7 +5,7 @@ require "stepmod/utils/change_edition_collection"
|
|
|
5
5
|
module Stepmod
|
|
6
6
|
module Utils
|
|
7
7
|
class Change
|
|
8
|
-
attr_accessor :schema_name
|
|
8
|
+
attr_accessor :schema_name, :mapping_table
|
|
9
9
|
attr_reader :change_editions
|
|
10
10
|
|
|
11
11
|
MODULE_TYPES = {
|
|
@@ -13,11 +13,20 @@ module Stepmod
|
|
|
13
13
|
mim: "mim",
|
|
14
14
|
arm_longform: "arm_lf",
|
|
15
15
|
mim_longform: "mim_lf",
|
|
16
|
+
mapping: "mapping",
|
|
17
|
+
changes: "changes",
|
|
18
|
+
mapping_table: "mapping",
|
|
16
19
|
}.freeze
|
|
17
20
|
|
|
21
|
+
TYPES_WITHOUT_EXTENSION = %w[
|
|
22
|
+
changes
|
|
23
|
+
mapping_table
|
|
24
|
+
].freeze
|
|
25
|
+
|
|
18
26
|
def initialize(stepmod_dir:, schema_name:, type:)
|
|
19
27
|
@stepmod_dir = stepmod_dir
|
|
20
28
|
@change_editions = Stepmod::Utils::ChangeEditionCollection.new
|
|
29
|
+
@mapping_table = {}
|
|
21
30
|
@schema_name = schema_name
|
|
22
31
|
@type = type
|
|
23
32
|
end
|
|
@@ -40,13 +49,24 @@ module Stepmod
|
|
|
40
49
|
alias_method :[], :fetch_change_edition
|
|
41
50
|
|
|
42
51
|
def save_to_file
|
|
43
|
-
|
|
52
|
+
change_hash = if @type == "mapping_table"
|
|
53
|
+
@mapping_table
|
|
54
|
+
else
|
|
55
|
+
to_h
|
|
56
|
+
end
|
|
57
|
+
return if change_hash.empty?
|
|
58
|
+
|
|
59
|
+
File.write(filepath(@type), Psych.dump(stringify_keys(change_hash)))
|
|
44
60
|
end
|
|
45
61
|
|
|
46
62
|
def to_h
|
|
63
|
+
change_editions_list = change_editions.to_h
|
|
64
|
+
|
|
65
|
+
return {} if change_editions_list.empty?
|
|
66
|
+
|
|
47
67
|
{
|
|
48
68
|
"schema" => schema_name,
|
|
49
|
-
"change_edition" =>
|
|
69
|
+
"change_edition" => change_editions_list,
|
|
50
70
|
}
|
|
51
71
|
end
|
|
52
72
|
|
|
@@ -58,10 +78,16 @@ module Stepmod
|
|
|
58
78
|
"data",
|
|
59
79
|
base_folder,
|
|
60
80
|
schema_name,
|
|
61
|
-
|
|
81
|
+
filename(type),
|
|
62
82
|
)
|
|
63
83
|
end
|
|
64
84
|
|
|
85
|
+
def filename(type)
|
|
86
|
+
return "#{MODULE_TYPES[type.to_sym]}.yaml" if TYPES_WITHOUT_EXTENSION.include?(type.to_s)
|
|
87
|
+
|
|
88
|
+
"#{MODULE_TYPES[type.to_sym] || schema_name}.changes.yaml"
|
|
89
|
+
end
|
|
90
|
+
|
|
65
91
|
def base_folder
|
|
66
92
|
if resource?
|
|
67
93
|
"resources"
|
|
@@ -69,6 +95,23 @@ module Stepmod
|
|
|
69
95
|
"modules"
|
|
70
96
|
end
|
|
71
97
|
end
|
|
98
|
+
|
|
99
|
+
# Hash#transform_keys is not available in Ruby 2.4
|
|
100
|
+
# so we have to do this ourselves :(
|
|
101
|
+
# symbolize hash keys
|
|
102
|
+
def stringify_keys(hash)
|
|
103
|
+
result = {}
|
|
104
|
+
hash.each_pair do |key, value|
|
|
105
|
+
result[key.to_s] = if value.is_a?(Hash)
|
|
106
|
+
stringify_keys(value)
|
|
107
|
+
elsif value.is_a?(Array)
|
|
108
|
+
value.map { |v| stringify_keys(v) }
|
|
109
|
+
else
|
|
110
|
+
value
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
result
|
|
114
|
+
end
|
|
72
115
|
end
|
|
73
116
|
end
|
|
74
117
|
end
|
|
@@ -2,7 +2,7 @@ module Stepmod
|
|
|
2
2
|
module Utils
|
|
3
3
|
class ChangeEdition
|
|
4
4
|
attr_accessor :version, :description
|
|
5
|
-
attr_reader :additions, :modifications, :deletions, :
|
|
5
|
+
attr_reader :additions, :modifications, :deletions, :changes
|
|
6
6
|
|
|
7
7
|
def initialize(options)
|
|
8
8
|
@version = options[:version]
|
|
@@ -10,7 +10,7 @@ module Stepmod
|
|
|
10
10
|
self.additions = options[:additions] || []
|
|
11
11
|
self.modifications = options[:modifications] || []
|
|
12
12
|
self.deletions = options[:deletions] || []
|
|
13
|
-
self.
|
|
13
|
+
self.changes = options[:changes] || []
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
def additions=(additions)
|
|
@@ -31,10 +31,10 @@ module Stepmod
|
|
|
31
31
|
@deletions = deletions
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
-
def
|
|
35
|
-
validate_type("
|
|
34
|
+
def changes=(changes)
|
|
35
|
+
validate_type("changes", changes, Array)
|
|
36
36
|
|
|
37
|
-
@
|
|
37
|
+
@changes = changes
|
|
38
38
|
end
|
|
39
39
|
|
|
40
40
|
def to_h
|
|
@@ -44,7 +44,7 @@ module Stepmod
|
|
|
44
44
|
"additions" => additions,
|
|
45
45
|
"modifications" => modifications,
|
|
46
46
|
"deletions" => deletions,
|
|
47
|
-
"
|
|
47
|
+
"changes" => changes,
|
|
48
48
|
}.reject { |_k, v| v.nil? || v.empty? }
|
|
49
49
|
end
|
|
50
50
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
require "nokogiri"
|
|
2
2
|
require "stepmod/utils/change_collection"
|
|
3
3
|
require "stepmod/utils/smrl_description_converter"
|
|
4
|
+
require_relative "parsers"
|
|
4
5
|
|
|
5
6
|
module Stepmod
|
|
6
7
|
module Utils
|
|
@@ -30,7 +31,11 @@ module Stepmod
|
|
|
30
31
|
all_modules_changes_files.each do |module_change_file|
|
|
31
32
|
xml_changes = Nokogiri::XML(File.read(module_change_file)).root
|
|
32
33
|
schema_name = Pathname.new(module_change_file).parent.basename.to_s
|
|
33
|
-
add_module_changes_to_collection(xml_changes, @collection,
|
|
34
|
+
add_module_changes_to_collection(xml_changes, @collection,
|
|
35
|
+
schema_name)
|
|
36
|
+
|
|
37
|
+
add_mapping_table_changes_to_collection(xml_changes, schema_name,
|
|
38
|
+
@collection)
|
|
34
39
|
end
|
|
35
40
|
|
|
36
41
|
@collection
|
|
@@ -38,6 +43,17 @@ module Stepmod
|
|
|
38
43
|
|
|
39
44
|
private
|
|
40
45
|
|
|
46
|
+
def add_mapping_table_changes_to_collection(xml_data, schema_name,
|
|
47
|
+
collection)
|
|
48
|
+
mapping_table = xml_data.at_xpath("//mapping_table")
|
|
49
|
+
return unless mapping_table
|
|
50
|
+
|
|
51
|
+
shale_object = Stepmod::Utils::Parsers::MappingTableParser.parse(mapping_table)
|
|
52
|
+
|
|
53
|
+
change = collection.fetch_or_initialize(schema_name, "mapping_table")
|
|
54
|
+
change.mapping_table = shale_object.to_hash
|
|
55
|
+
end
|
|
56
|
+
|
|
41
57
|
# rubocop:disable Metrics/MethodLength
|
|
42
58
|
def add_resource_changes_to_collection(xml_data, collection)
|
|
43
59
|
xml_data.xpath("//changes").each do |changes_node|
|
|
@@ -64,9 +80,12 @@ module Stepmod
|
|
|
64
80
|
options = {
|
|
65
81
|
schema_name: schema_name,
|
|
66
82
|
version: change_node.attr("version"),
|
|
67
|
-
description: converted_description(change_node.xpath("description").first),
|
|
83
|
+
# description: converted_description(change_node.xpath("description").first),
|
|
68
84
|
}
|
|
69
85
|
|
|
86
|
+
add_mapping_changes(collection, change_node, options)
|
|
87
|
+
add_changes(collection, change_node, options)
|
|
88
|
+
|
|
70
89
|
MODULE_TYPES.each do |type|
|
|
71
90
|
add_module_changes(
|
|
72
91
|
collection,
|
|
@@ -97,21 +116,34 @@ module Stepmod
|
|
|
97
116
|
schema_name = options[:schema_name]
|
|
98
117
|
change = collection.fetch_or_initialize(schema_name, type)
|
|
99
118
|
|
|
119
|
+
return if changes.nil?
|
|
120
|
+
|
|
100
121
|
change_edition = extract_change_edition(changes, options)
|
|
101
122
|
change.add_change_edition(change_edition)
|
|
102
|
-
|
|
103
|
-
if type == "arm"
|
|
104
|
-
add_mapping_changes(collection, change_node, options)
|
|
105
|
-
end
|
|
106
123
|
end
|
|
107
124
|
|
|
108
125
|
def add_mapping_changes(collection, change_node, options)
|
|
126
|
+
mapping_changes = extract_mapping_changes(change_node)
|
|
127
|
+
return if mapping_changes.empty?
|
|
128
|
+
|
|
109
129
|
change_edition = collection
|
|
110
|
-
.fetch_or_initialize(options[:schema_name], "
|
|
130
|
+
.fetch_or_initialize(options[:schema_name], "mapping")
|
|
111
131
|
.change_editions
|
|
112
132
|
.fetch_or_initialize(options[:version])
|
|
113
133
|
|
|
114
|
-
change_edition.
|
|
134
|
+
change_edition.changes = mapping_changes
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def add_changes(collection, change_node, options)
|
|
138
|
+
description = converted_description(change_node.xpath("description").first)
|
|
139
|
+
return if description.nil? || description.empty?
|
|
140
|
+
|
|
141
|
+
change_edition = collection
|
|
142
|
+
.fetch_or_initialize(options[:schema_name], "changes")
|
|
143
|
+
.change_editions
|
|
144
|
+
.fetch_or_initialize(options[:version])
|
|
145
|
+
|
|
146
|
+
change_edition.description = description
|
|
115
147
|
end
|
|
116
148
|
|
|
117
149
|
def extract_mapping_changes(change_node)
|
|
@@ -119,17 +151,21 @@ module Stepmod
|
|
|
119
151
|
|
|
120
152
|
change_node.xpath("mapping.changes").each do |changes|
|
|
121
153
|
changes.xpath("mapping.change").each do |change|
|
|
122
|
-
mappings << { "change" => change.text }
|
|
154
|
+
mappings << { "change" => clean_string(change.text.strip) }
|
|
123
155
|
end
|
|
124
156
|
|
|
125
157
|
changes.xpath("description").each do |change|
|
|
126
|
-
mappings << { "description" => change.text }
|
|
158
|
+
mappings << { "description" => clean_string(change.text.strip) }
|
|
127
159
|
end
|
|
128
160
|
end
|
|
129
161
|
|
|
130
162
|
mappings
|
|
131
163
|
end
|
|
132
164
|
|
|
165
|
+
def clean_string(string)
|
|
166
|
+
string.gsub(/[\n\t]/, " ").squeeze(" ")
|
|
167
|
+
end
|
|
168
|
+
|
|
133
169
|
def extract_change_edition(schema_changes, options)
|
|
134
170
|
type = options[:type] || "schema"
|
|
135
171
|
node_type = type.gsub("_longform", "")
|
|
@@ -17,7 +17,8 @@ module Stepmod
|
|
|
17
17
|
# TODO: converted_definition is not supposed to be an attribute, it is
|
|
18
18
|
# supposed to be a method!
|
|
19
19
|
class << self
|
|
20
|
-
def parse(definition_xml, reference_anchor:, reference_clause:,
|
|
20
|
+
def parse(definition_xml, reference_anchor:, reference_clause:,
|
|
21
|
+
file_path:, language_code: "eng")
|
|
21
22
|
converted_definition = Stepmod::Utils::StepmodDefinitionConverter.convert(
|
|
22
23
|
definition_xml,
|
|
23
24
|
{
|
|
@@ -64,14 +65,15 @@ module Stepmod
|
|
|
64
65
|
end
|
|
65
66
|
|
|
66
67
|
term = Stepmod::Utils::Converters::Term
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
68
|
+
.new
|
|
69
|
+
.convert(
|
|
70
|
+
definition_xml.xpath(".//term").first,
|
|
71
|
+
)
|
|
71
72
|
|
|
72
73
|
# [4..-1] because we want to skip the initial `=== {title}`
|
|
73
74
|
designations = [
|
|
74
|
-
{ "designation" => term[4..-1], "type" => "expression",
|
|
75
|
+
{ "designation" => term[4..-1], "type" => "expression",
|
|
76
|
+
"normative_status" => "preferred" },
|
|
75
77
|
]
|
|
76
78
|
|
|
77
79
|
alts.each do |alt|
|
|
@@ -84,10 +86,10 @@ module Stepmod
|
|
|
84
86
|
def definition_xml_definition(definition_xml, reference_anchor)
|
|
85
87
|
# We reject the <p> that was considered an alternative term (length<=20)
|
|
86
88
|
text_nodes = definition_xml
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
89
|
+
.xpath(".//def")
|
|
90
|
+
.first
|
|
91
|
+
.children
|
|
92
|
+
.reject { |n| n.name == "p" && n.text.length <= 20 }
|
|
91
93
|
|
|
92
94
|
wrapper = "<def>#{text_nodes.map(&:to_s).join}</def>"
|
|
93
95
|
|