stepmod-utils 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc937fce5f5bcc0ae84e007894542cd6bcf76f9557436f8543bbfbd6e79d7d8c
-  data.tar.gz: 427400b1a2353284fbdd080ed4219afa59d5230b5f297a63615eef58bc82cde6
+  metadata.gz: cb309548a8dd492982004515c2d11e45b72e4043452e72e809355dc9a8ffa7fd
+  data.tar.gz: 27510963bd60fb68b0a63286bc7dd11efbaf3a780c3fcb0227c8934a550f830d
 SHA512:
-  metadata.gz: 9fcf8e5182b797372d2d87d55ddcda604255bfea85efb22108a1e8cdb6e3b15091cde5ab8ef33f3d25b7ee4855029719b7409202f959cf9c6fe14a3c56b2e4a7
-  data.tar.gz: cf4db46480fbac65e538f09a5edfb1ebca7ee15fdb5d8020d131c615d85a8ed4f978454486299fd5696811c1b89632cf3c58ca86401032798fa1ece49ba053ff
+  metadata.gz: efc594b7dab3677aa2ada861d787abd7078b222de4b6e7c5463549a4ce8c7e511bac6266e8973e3904e0dbbd4be2eb864f15cd817a172811cee5ec866783f143
+  data.tar.gz: b3d08ac1688b6a0e0a6aafd64b7f19339fef0414f43848a84fa6a3503300c457ea52a5d585ffdd7d77a85798725a3817d54c9d032767de26c48cdfac30089870

data/.github/workflows/test-concept-generation.yml

@@ -27,7 +27,10 @@ jobs:
           path: iso-10303-stepmod-wg12
 
       - name: Generate Annotated EXPRESS files
-        run: bundle exec stepmod-annotate-all ./iso-10303-stepmod-wg12
+        run: |
+          bundle exec stepmod-annotate-all \
+            --stepmod-dir ./iso-10303-stepmod-wg12 \
+            --output schemas
 
       - name: Generate Concept YAML files
         working-directory: iso-10303-stepmod-wg12

data/README.adoc

@@ -25,10 +25,106 @@ Or include it in your gemspec.
 
 == Usage
 
+=== EXPRESS schema validation
+
+This command uses Expressir to validate all EXPRESS schemas listed in the
+`{stepmod}/repository_index.xml` file.
+
+There is a `SHOULD_REWRITE` constant that can be switched in the code which
+allows overwriting of the EXPRESS schemas into a pretty-printed format.
+
+WARNING: The `SHOULD_REWRITE` function is turned off because Expressir currently
+does not support outputting remarks, and some of the pretty-printed schemas will
+fail validation.
+
+[source,sh]
+----
+$ stepmod-validate-express {stepmod-directory}
+----
+
+Output:
+
+[source]
+----
+Validating all EXPRESS schema files listed in:
+repository_index.xml
+
+NOTE: SHOULD_REWRITE mode is turned OFF
+validating: ./data/modules/activity/arm.exp
+validated : ./data/modules/activity/arm.exp
+validating: ./data/modules/activity/mim.exp
+validated : ./data/modules/activity/mim.exp
+validating: ./data/modules/activity_as_realized/arm.exp
+validated : ./data/modules/activity_as_realized/arm.exp
+validating: ./data/modules/activity_as_realized/mim.exp
+validated : ./data/modules/activity_as_realized/mim.exp
+...
+The following schemas failed validation:
+SCHEMA: data/resources/geometric_model_schema/geometric_model_schema.exp
+----
+
+=== Generating Annotated EXPRESS
+
+This command takes the STEPmod directory and parses its XML to synthesize
+with the target EXPRESS schemas to migrate into Annotated EXPRESS schemas.
+
+Given the following schemas:
+
+* Simple EXPRESS schema: `{stepmod-root-dir}/data/resources/action_schema/action_schema.exp`
+* Output of Annotated EXPRESS schema: `{schemas-output-dir}/action_schema/action_schema.exp`
+
+Command:
+
+[source,sh]
+----
+$ bundle exec stepmod-annotate-all \
+  --stepmod-dir {stepmod-root-dir} \
+  --output {schemas-output-dir}
+----
+
+e.g.
+
+[source,sh]
+----
+$ bundle exec stepmod-annotate-all \
+  --stepmod-dir ./iso-10303-stepmod-wg12 \
+  --output schemas
+----
+
+
+Output:
+
 [source,sh]
 ----
-# Extracts from specified stepmod/data/ directory
-$ stepmod-extract-terms -p {stepmod-data-directory} -i {path-to-repository_index.xml}
+$ bundle exec stepmod-annotate-all --stepmod-dir iso-10303-stepmod-wg12 --output schemas
+[stepmod-utils] STEPmod data path: `iso-10303-stepmod-wg12`
+[stepmod-utils] SRL schemas output path: `schemas`
+5760: `iso-10303-stepmod-wg12/data/modules/activity/arm.exp` processing...
+5760: Written to: `schemas/activity/arm.exp`
+5760: Done processing `arm.exp` => `schemas/activity/arm.exp`
+5760: `iso-10303-stepmod-wg12/data/modules/activity/mim.exp` processing...
+5760: Written to: `schemas/activity/mim.exp`
+5760: Done processing `mim.exp` => `schemas/activity/mim.exp`
+5760: `iso-10303-stepmod-wg12/data/modules/activity_as_realized/arm.exp` processing...
+5760: Written to: `schemas/activity_as_realized/arm.exp`
+5760: Done processing `arm.exp` => `schemas/activity_as_realized/arm.exp`
+5760: `iso-10303-stepmod-wg12/data/modules/activity_as_realized/mim.exp` processing...
+5760: Written to: `schemas/activity_as_realized/mim.exp`
+5760: Done processing `mim.exp` => `schemas/activity_as_realized/mim.exp`
+5760: `iso-10303-stepmod-wg12/data/modules/activity_characterized/arm.exp` processing...
+5760: Written to: `schemas/activity_characterized/arm.exp`
+...
+----
+
+
+=== ISO 10303-2 concepts extraction
+
+[source,sh]
+----
+$ bundle exec stepmod-extract-concepts \
+  -p ./data \
+  -i ./repository_index.xml \
+  -o ./output-yaml
 ----
 
 Then these files will be created:
@@ -36,6 +132,130 @@ Then these files will be created:
 * `031-generated-terms.adoc` all terms extracted
 * `991-generated-bibliography.adoc` all bibliographic sources where the terms come from
 
+If the STEPmod repository is a Git repository, term extract will work.
+
+Sample output:
+
+[source,sh]
+----
+[stepmod-utils] INFO: STEPmod directory set to ../iso-10303-stepmod.
+[stepmod-utils] INFO: Detecting paths...
+[stepmod-utils] INFO: Processing XML file data/application_protocols/boundary_representation_for_iso_14306_open_jt/application_protocol.xml
+[stepmod-utils] INFO: skipped ISO/CD 10303-243 as it is not IS, DIS or TS
+[stepmod-utils] INFO: Processing XML file data/application_protocols/configuration_control_3d_design_ed2/application_protocol.xml
+[stepmod-utils] INFO: Completed processing XML file data/application_protocols/configuration_control_3d_design_ed2/application_protocol.xml
+[stepmod-utils] INFO: Processing XML file data/application_protocols/electronic_assembly_interconnect_and_packaging_design/application_protocol.xml
+[stepmod-utils] INFO: skipped ISO/CD 10303-210:2019 as it is not IS, DIS or TS
+[stepmod-utils] INFO: Processing XML file data/application_protocols/functional_data_and_their_schematic_representation_for_process_plant/application_protocol.xml
+[stepmod-utils] INFO: Completed processing XML file data/application_protocols/functional_data_and_their_schematic_representation_for_process_plant/application_protocol.xml
+...
+----
+
+
+
+=== Creating the SRL
+
+The single command to create the SRL from the STEPmod repository is as follows.
+
+The command also takes the `--stepmod2mn-path {path}` option to specify the
+location of the `stepmod2mn.jar` file. If it is not specified, or the path
+doesn't exist, the command will automatically download the latest release of it
+to use.
+
+[source,sh]
+----
+$ bundle exec stepmod-srl-migrate \
+    --srl-schemas-dir {path-to-new-srl-schemas} \
+    --srl-docs-dir {path-to-new-srl-documents} \
+    --stepmod-dir {stepmod-root-path}
+----
+
+e.g.
+
+[source,sh]
+----
+$ bundle exec stepmod-srl-migrate \
+    --srl-schemas-dir schemas \
+    --srl-docs-dir documents \
+    --stepmod-dir iso-10303-stepmod-wg12
+----
+
+Output:
+
+[source,sh]
+----
+To use retry middleware with Faraday v2.0+, install `faraday-retry` gem
+[stepmod-utils] `stepmod2mn.jar` path: `/Users/mulgogi/src/mn/iso-10303-srl/stepmod2mn-1.25.jar`
+[stepmod-utils] STEPmod path: `iso-10303-stepmod-wg12`
+[stepmod-utils] SRL output schemas directory path: `schemas`
+[stepmod-utils] SRL output documents directory path: `documents`
+[stepmod-utils] ******************************
+[stepmod-utils] [SRL MIGRATION] Migration starts!
+[stepmod-utils] ******************************
+[stepmod-utils] [SRL MIGRATION: stepmod2mn.jar] START document migration and SVG generation.
+[stepmod-utils] ******************************
+stepmod2mn Generate SVG file for iso-10303-stepmod-wg12/data/resource_docs/systems_engineering_representation/schema_diagexpg1.xml...
+SVG saved in documents/systems_engineering_representation/schema_diagexpg1.svg.
+Generate SVG file for iso-10303-stepmod-wg12/data/resource_docs/fundamentals_of_product_description_and_support/schema_diagexpg1.xml...
+SVG saved in documents/fundamentals_of_product_description_and_support/schema_diagexpg1.svg.
+Generate SVG file for iso-10303-stepmod-wg12/data/resource_docs/mathematical_representation/schema_diagexpg1.xml...
+...
+Generate SVG file for iso-10303-stepmod-wg12/data/modules/assembly_module_with_subassembly/mimexpg2.xml...
+SVG saved in documents/assembly_module_with_subassembly/mimexpg2.svg.
+End!
+[stepmod-utils] ******************************
+[stepmod-utils] [SRL MIGRATION: stepmod2mn.jar] COMPLETE document migration and SVG generation.
+[stepmod-utils] ******************************
+[stepmod-utils] [SRL MIGRATION: stepmod-utils] START Annotated EXPRESS generation.
+[stepmod-utils] ******************************
+[stepmod-utils] STEPmod data path: `iso-10303-stepmod-wg12`
+[stepmod-utils] SRL schemas output path: `schemas`
+5720: `iso-10303-stepmod-wg12/data/modules/activity/arm.exp` processing...
+5720: Written to: `schemas/activity/arm.exp`
+5720: Done processing `arm.exp` => `schemas/activity/arm.exp`
+5720: `iso-10303-stepmod-wg12/data/modules/activity/mim.exp` processing...
+5720: Written to: `schemas/activity/mim.exp`
+5720: Done processing `mim.exp` => `schemas/activity/mim.exp`
+5720: `iso-10303-stepmod-wg12/data/modules/activity_as_realized/arm.exp` processing...
+...
+5720: Done processing `mim.exp` => `schemas/zonal_breakdown/mim.exp`
+5720: `iso-10303-stepmod-wg12/data/resources/action_and_model_relationships_schema/action_and_model_relationships_schema.exp` processing...
+5720: Written to: `schemas/action_and_model_relationships_schema/action_and_model_relationships_schema.exp`
+5720:     Copy image SKIPPED for `schemas/action_and_model_relationships_schema/action_and_model_relationships_schemaexpg1.svg` as it exists.
+5720:     Copy image SKIPPED for `schemas/action_and_model_relationships_schema/action_and_model_relationships_schemaexpg2.svg` as it exists.
+5720: Done processing `action_and_model_relationships_schema.exp` => `schemas/action_and_model_relationships_schema/action_and_model_relationships_schema.exp`
+5720: `iso-10303-stepmod-wg12/data/resources/action_schema/action_schema.exp` processing...
+5720: Written to: `schemas/action_schema/action_schema.exp`
+5720:     Copy image SKIPPED for `schemas/action_schema/action_schemaexpg1.svg` as it exists.
+5720:     Copy image SKIPPED for `schemas/action_schema/action_schemaexpg2.svg` as it exists.
+5720: Done processing `action_schema.exp` => `schemas/action_schema/action_schema.exp`
+5720: `iso-10303-stepmod-wg12/data/resources/aic_advanced_brep/aic_advanced_brep.exp` processing...
+5720: Written to: `schemas/aic_advanced_brep/aic_advanced_brep.exp`
+5720: Done processing `aic_advanced_brep.exp` => `schemas/aic_advanced_brep/aic_advanced_brep.exp`
+5720: `iso-10303-stepmod-wg12/data/resources/aic_associative_draughting_elements/aic_associative_draughting_elements.exp` processing...
+5720: Written to: `schemas/aic_associative_draughting_elements/aic_associative_draughting_elements.exp`
+...
+5720: Written to: `schemas/business_object_models/managed_model_based_3d_engineering_domain/DomainModel.exp`
+5720: Done processing `DomainModel.exp` => `schemas/business_object_models/managed_model_based_3d_engineering_domain/DomainModel.exp`
+[stepmod-utils] ******************************
+[stepmod-utils] [SRL MIGRATION: stepmod-utils] COMPLETE Annotated EXPRESS generation.
+[stepmod-utils] ******************************
+[stepmod-utils] [SRL MIGRATION: stepmod-utils] START EXPRESS change YAML extraction.
+[stepmod-utils] STEPmod data path: `iso-10303-stepmod-wg12`
+[stepmod-utils] Copied arm.changes.yaml to schemas/activity/arm.changes.yaml
+[stepmod-utils] Copied arm_lf.changes.yaml to schemas/activity/arm_lf.changes.yaml
+[stepmod-utils] Copied mim.changes.yaml to schemas/activity/mim.changes.yaml
+[stepmod-utils] Copied mim_lf.changes.yaml to schemas/activity/mim_lf.changes.yaml
+[stepmod-utils] Copied arm.changes.yaml to schemas/activity_as_realized/arm.changes.yaml
+...
+[stepmod-utils] Copied topology_schema.changes.yaml to schemas/topology_schema/topology_schema.changes.yaml
+[stepmod-utils] ******************************
+[stepmod-utils] [SRL MIGRATION: stepmod-utils] COMPLETE EXPRESS change YAML extraction.
+[stepmod-utils] ******************************
+[stepmod-utils] [SRL MIGRATION] Migration complete!
+[stepmod-utils] ******************************
+----
+
 == Conversion of STEPmod descriptions.xml into adoc
 
 [source,sh]
@@ -111,7 +331,18 @@ expg_image:contract_schemaexpg1.xml[]
 expg_image:contract_schemaexpg2.xml[]
 ----
 
-== Convertion of all schema history information into independent per-schema YAML files
+== Generate EXPRESS Change YAML files
+
+EXPRESS Change YAML files are YAML files that contain schema history
+information.
+
+This command migrates existing schema change information (in XML) into
+independent per-schema YAML files.
+
+The files will be created in the same directory as the schema:
+
+* Schema: `shape_dimension_schema/shape_dimension_schema.exp`
+* Output: `shape_dimension_schema/shape_dimension_schema.changes.yaml`
 
 Command:
 
@@ -277,25 +508,3 @@ change_edition:
     name: connected_edge_with_length_set_items
 ----
 
-
-== Git repository
-
-If the STEPmod repository is a Git repository, term extract will work.
-
-Sample output:
-
-[source,sh]
-----
-[stepmod-utils] INFO: STEPmod directory set to ../iso-10303-stepmod.
-[stepmod-utils] INFO: Detecting paths...
-[stepmod-utils] INFO: Processing XML file data/application_protocols/boundary_representation_for_iso_14306_open_jt/application_protocol.xml
-[stepmod-utils] INFO: skipped ISO/CD 10303-243 as it is not IS, DIS or TS
-[stepmod-utils] INFO: Processing XML file data/application_protocols/configuration_control_3d_design_ed2/application_protocol.xml
-[stepmod-utils] INFO: Completed processing XML file data/application_protocols/configuration_control_3d_design_ed2/application_protocol.xml
-[stepmod-utils] INFO: Processing XML file data/application_protocols/electronic_assembly_interconnect_and_packaging_design/application_protocol.xml
-[stepmod-utils] INFO: skipped ISO/CD 10303-210:2019 as it is not IS, DIS or TS
-[stepmod-utils] INFO: Processing XML file data/application_protocols/functional_data_and_their_schematic_representation_for_process_plant/application_protocol.xml
-[stepmod-utils] INFO: Completed processing XML file data/application_protocols/functional_data_and_their_schematic_representation_for_process_plant/application_protocol.xml
-...
-----
-

data/exe/stepmod-annotate-all

@@ -4,8 +4,52 @@ require "concurrent"
 require "stepmod/utils/stepmod_file_annotator"
 require "fileutils"
 
-stepmod_dir = ARGV.first || Dir.pwd
-stepmod_dir = Pathname.new(stepmod_dir.gsub("\\", "/"))
+require "bundler/setup"
+require "optparse"
+
+def log(message, indent = 0)
+  indent_spaces = " " * (indent * 2)
+  puts "[stepmod-utils] #{indent_spaces}#{message}"
+end
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.on(
+    "--stepmod-dir STEPMOD_DIR",
+    String,
+    "Path to STEPmod directory",
+  ) do |path|
+    options[:stepmod_dir] = path
+  end
+
+  opts.on(
+    "--output SRL_SCHEMAS_DIR",
+    String,
+    "Path to output SRL schemas",
+  ) do |path|
+    options[:srl_output_schemas_dir] = path
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+stepmod_dir = options[:stepmod_dir]
+raise StandardError.new("--stepmod-dir not set") unless stepmod_dir
+
+stepmod_path = Pathname.new(stepmod_dir)
+unless (stepmod_path.join("data").exist? && stepmod_path.join("data").directory?)
+  raise StandardError.new("--stepmod-dir should be the root STEPmod directory and contains data/")
+end
+log "STEPmod data path: `#{stepmod_dir}`"
+
+srl_output_schemas_dir = options[:srl_output_schemas_dir]
+raise StandardError.new("--output directory for SRL schemas not set") unless srl_output_schemas_dir
+log "SRL schemas output path: `#{srl_output_schemas_dir}`"
 
 def all_express_files(stepmod_dir)
   index_file = File.read(File.join(stepmod_dir, "repository_index.xml"))
@@ -46,7 +90,6 @@ end
 # It will return nil if the file does not exist
 def file_system_path(filepath, filename_cache)
   dir = File.dirname(filepath)
-  filename = File.basename(filepath)
 
   return filename_cache[dir][filepath.downcase] if filename_cache[dir]
 
@@ -69,32 +112,50 @@ MAX_QUEUE_SIZE = MAX_THREADS * 4
 files = all_express_files(stepmod_dir)
 filename_cache = {}
 
+FileUtils.mkdir_p(srl_output_schemas_dir)
+
 files.each do |file|
-  puts "#{Thread.current.object_id}: Processing #{file}"
+  puts "#{Thread.current.object_id}: `#{file}` processing..."
 
   result = Stepmod::Utils::StepmodFileAnnotator.new(
     express_file: file,
     stepmod_dir: stepmod_dir,
   ).call
 
-  annotated_file_name = "#{File.basename(file, '.exp')}_annotated.exp"
-  annotated_file_path = File.join(File.dirname(file), annotated_file_name)
+  destination_rel_path = Pathname.new(file).dirname.relative_path_from(stepmod_path.join("data"))
+  directory_type = destination_rel_path.each_filename.to_a.first
+
+  if %w(resources modules).include?(directory_type)
+    destination_rel_path = Pathname.new(file).dirname.relative_path_from(stepmod_path.join("data").join(directory_type))
+  end
+
+  destination_path = Pathname.new(srl_output_schemas_dir).join(destination_rel_path)
+  FileUtils.mkdir_p(destination_path)
+
+  annotated_file_path = destination_path.join(File.basename(file))
 
   File.open(annotated_file_path, "w") do |annotated_file|
     annotated_file.puts(result[:annotated_text])
   end
+  puts "#{Thread.current.object_id}: Written to: `#{annotated_file_path}`"
 
   result[:images_references].each do |source, destination|
-    source_path = File.join(stepmod_dir, "data", source)
-    system_source_path = file_system_path(source_path, filename_cache)
-    destination_path = File.join(File.dirname(file), destination)
-
-    puts "#{Thread.current.object_id}:     Starting to copy file from #{source_path} to #{destination_path}"
-    next if !system_source_path
-
-    FileUtils.cp(system_source_path, destination_path)
-    puts "#{Thread.current.object_id}:     Done copying #{system_source_path} to #{destination_path}"
+    source_path = stepmod_path.join("data", source)
+    system_source_path = file_system_path(source_path.to_s, filename_cache)
+    image_dest_path = destination_path.join(destination)
+
+    unless system_source_path
+      if image_dest_path.exist?
+        puts "#{Thread.current.object_id}:     Copy image SKIPPED for `#{image_dest_path}` as it exists."
+      else
+        puts "#{Thread.current.object_id}:     Copy image FAILED from `#{source_path}` to `#{image_dest_path}`"
+      end
+      next
+    end
+
+    FileUtils.cp(system_source_path, image_dest_path)
+    puts "#{Thread.current.object_id}:     Copy image SUCCESS: `#{system_source_path}` to `#{image_dest_path}`"
   end
 
-  puts "#{Thread.current.object_id}: Done processing #{File.basename(file)} => #{annotated_file_path}."
+  puts "#{Thread.current.object_id}: Done processing `#{File.basename(file)}` => `#{annotated_file_path}`"
 end

data/exe/stepmod-convert-express-description

@@ -8,15 +8,6 @@ bin_file = Pathname.new(__FILE__).realpath
 # add self to libpath
 $:.unshift File.expand_path("../../lib", bin_file)
 
-# Fixes https://github.com/rubygems/rubygems/issues/1420
-require "rubygems/specification"
-
-module Gem
-  class Specification
-    def this; self; end
-  end
-end
-
 require "bundler/setup"
 require "stepmod/utils/smrl_description_converter"
 

data/exe/stepmod-convert-express-resource

@@ -8,15 +8,6 @@ bin_file = Pathname.new(__FILE__).realpath
 # add self to libpath
 $:.unshift File.expand_path("../../lib", bin_file)
 
-# Fixes https://github.com/rubygems/rubygems/issues/1420
-require "rubygems/specification"
-
-module Gem
-  class Specification
-    def this; self; end
-  end
-end
-
 require "bundler/setup"
 require "stepmod/utils/smrl_resource_converter"
 

data/exe/stepmod-extract-changes

@@ -7,15 +7,6 @@ bin_file = Pathname.new(__FILE__).realpath
 # add self to libpath
 $:.unshift File.expand_path("../../lib", bin_file)
 
-# Fixes https://github.com/rubygems/rubygems/issues/1420
-require "rubygems/specification"
-
-module Gem
-  class Specification
-    def this; self; end
-  end
-end
-
 require "bundler/setup"
 require "stepmod/utils/changes_extractor"
 require "optparse"

data/exe/stepmod-extract-concepts

@@ -9,15 +9,6 @@ bin_file = Pathname.new(__FILE__).realpath
 # add self to libpath
 $:.unshift File.expand_path("../../lib", bin_file)
 
-# Fixes https://github.com/rubygems/rubygems/issues/1420
-require "rubygems/specification"
-
-module Gem
-  class Specification
-    def this; self; end
-  end
-end
-
 require "bundler/setup"
 require_relative "../lib/stepmod/utils/terms_extractor"
 require "optparse"

data/exe/stepmod-srl-migrate

@@ -3,35 +3,43 @@
 # encoding: UTF-8
 
 require "pathname"
+require "fileutils"
 bin_file = Pathname.new(__FILE__).realpath
 
 # add self to libpath
 $:.unshift File.expand_path("../../lib", bin_file)
 
-# Fixes https://github.com/rubygems/rubygems/issues/1420
-require "rubygems/specification"
-
-module Gem
-  class Specification
-    def this; self; end
-  end
-end
-
 require "bundler/setup"
 require "optparse"
 
-def log(message)
+def log(message, indent = 0)
+  indent_spaces = " " * (indent * 2)
   puts "[stepmod-utils] #{message}"
 end
 
+def copy_files_to_schemas(path, srl_output_docs_dir)
+  schema_name = path.dirname.basename
+  file_name = path.basename
+
+  new_dir = File.join(srl_output_docs_dir, schema_name)
+  new_file_path = File.join(new_dir, file_name)
+
+  unless Dir.exist?(new_dir)
+    FileUtils.mkdir_p(new_dir)
+  end
+
+  FileUtils.copy_file(path, new_file_path)
+  log("Copied #{path.basename} to #{new_file_path}", 1)
+end
+
 options = {}
 OptionParser.new do |opts|
   opts.banner = "Usage: #{$0} [options]"
 
   opts.on(
-    "--stepmod-data-dir STEPMOD_DATA_DIR",
+    "--stepmod-dir STEPMOD_DIR",
     String,
-    "Path to STEPmod data directory",
+    "Path to the STEPmod root directory",
   ) do |path|
     options[:stepmod_dir] = path
   end
@@ -39,7 +47,7 @@ OptionParser.new do |opts|
   opts.on(
     "--srl-docs-dir SRL_DOCS_DIR",
     String,
-    "Path to output srl docs",
+    "Path to output SRL docs",
   ) do |path|
     options[:srl_output_docs_dir] = path
   end
@@ -47,50 +55,118 @@ OptionParser.new do |opts|
   opts.on(
     "--srl-schemas-dir SRL_SCHEMAS_DIR",
     String,
-    "Path to output srl schemas",
+    "Path to output SRL schemas",
   ) do |path|
     options[:srl_output_schemas_dir] = path
   end
 
   opts.on(
-    "-s",
-    "--stepmod2mn-executeable STEPMOD2MN_EXECUTEABLE",
+    "--stepmod2mn-path STEPMOD2MN_JAR_PATH",
     String,
-    "Path to STEPMOD2MN_EXECUTEABLE",
+    "Path to stepmod2mn-vx.y.jar",
   ) do |path|
     options[:stepmod2mn_path] = path
   end
 
-  opts.on_tail("-h", "--help", "Show this message") do
+  opts.on_tail("-h", "--help", "Show this help message") do
     puts opts
     exit
   end
 end.parse!
 
+def download_stepmod2mn(path)
+  return path if path && path.exist?
+
+  require 'octokit'
+  require "down"
+  require "fileutils"
+
+  client = Octokit::Client.new
+  download_url = client.latest_release('metanorma/stepmod2mn').assets.first.browser_download_url
+
+  filename = Pathname.new(download_url).basename.to_s
+
+  destination_path = path || Pathname.pwd.join(filename)
+  return destination_path if !destination_path.nil? && destination_path.exist?
+
+  puts "Downloading `stepmod2mn.jar` to `#{destination_path}`."
+  tempfile = Down.download(download_url)
+  FileUtils.mkdir_p(Pathname.new(destination_path).dirname)
+  FileUtils.mv(tempfile.path, destination_path)
+
+  destination_path
+end
+
 stepmod2mn_path = options[:stepmod2mn_path]
-if stepmod2mn_path.nil?
-  raise StandardError.new(
-    "stepmod2mn executeable is required, set with the `-s` option.",
-  )
+if !stepmod2mn_path.nil?
+  if Pathname.new(stepmod2mn_path).exist?
+    puts "Specified `stepmod2mn.jar` found, using existing file."
+  else
+    stepmod2mn_path = download_stepmod2mn(Pathname.new(stepmod2mn_path))
+  end
+  puts "Specified `stepmod2mn.jar` not found, downloading from GitHub."
+else
+  stepmod2mn_path = download_stepmod2mn(nil)
 end
+log "`stepmod2mn.jar` path: `#{stepmod2mn_path}`"
 
 stepmod_dir = options[:stepmod_dir] || "stepmod"
 if stepmod_dir.nil?
   raise StandardError.new(
-    "stepmod data directory required, set with the `--stepmod_dir` option.",
+    "Please provide path to the STEPmod root directory with `--stepmod_dir`.",
   )
 end
 
-migrate_docs_command = %(stepmod-srl-migrate-docs -s "#{stepmod2mn_path}" --stepmod-data-dir "#{stepmod_dir}")
-migrate_docs_command += %( --srl-docs-dir #{options[:srl_output_docs_dir]}) if options[:srl_output_docs_dir]
+stepmod_path = Pathname.new(stepmod_dir)
+unless stepmod_path.exist? && stepmod_path.directory? &&
+  stepmod_path.join("data").exist? && stepmod_path.join("data").directory?
+  puts "Specified STEPmod directory not found, not a directory, or does not contain `data/`. Aborting."
+  exit 1
+end
+
+stepmod_dir = options[:stepmod_dir] || "stepmod"
+log "STEPmod path: `#{stepmod_dir}`"
 
-migrate_schemas_command = %(stepmod-srl-migrate-schemas -s "#{stepmod2mn_path}" --stepmod-data-dir "#{stepmod_dir}")
-migrate_schemas_command += %( --srl-schemas-dir #{options[:srl_output_schemas_dir]}) if options[:srl_output_schemas_dir]
+srl_output_schemas_dir = options[:srl_output_schemas_dir] || "schemas"
+unless File.directory?(srl_output_schemas_dir)
+  FileUtils.mkdir_p(srl_output_schemas_dir)
+end
+log "SRL output schemas directory path: `#{srl_output_schemas_dir}`"
+
+srl_output_docs_dir = options[:srl_output_docs_dir] || "documents"
+unless File.directory?(srl_output_docs_dir)
+  FileUtils.mkdir_p(srl_output_docs_dir)
+end
+log "SRL output documents directory path: `#{srl_output_docs_dir}`"
+
+log("*"*30)
+log("[SRL MIGRATION] Migration starts!")
+
+log("*"*30)
+log("[SRL MIGRATION: stepmod2mn.jar] START document migration and SVG generation.")
+log("*"*30)
+system "java -Xss5m -jar #{stepmod2mn_path} #{stepmod_dir} --output #{srl_output_docs_dir} --svg"
+log("*"*30)
+log("[SRL MIGRATION: stepmod2mn.jar] COMPLETE document migration and SVG generation.")
+
+log("*"*30)
+log("[SRL MIGRATION: stepmod-utils] START Annotated EXPRESS generation.")
+log("*"*30)
+system "stepmod-annotate-all --stepmod-dir #{stepmod_dir} --output #{srl_output_schemas_dir}"
+log("*"*30)
+log("[SRL MIGRATION: stepmod-utils] COMPLETE Annotated EXPRESS generation.")
+
+log("*"*30)
+log("[SRL MIGRATION: stepmod-utils] START EXPRESS change YAML extraction.")
+system "stepmod-extract-changes -p #{stepmod_dir}"
+Dir.glob("#{stepmod_dir}/data/**/*.changes.yaml").each do |filepath|
+  path = Pathname.new(filepath)
+  copy_files_to_schemas(path, srl_output_schemas_dir)
+end
+log("*"*30)
+log("[SRL MIGRATION: stepmod-utils] COMPLETE EXPRESS change YAML extraction.")
 
-log("--- START Migrate Docs ---")
-system migrate_docs_command
-log("--- END Migrate Docs ---")
+log("*"*30)
+log("[SRL MIGRATION] Migration complete!")
 
-log("--- START Migrate Schemas ---")
-system migrate_schemas_command
-log("--- END Migrate Schemas ---")
+log("*"*30)

data/exe/stepmod-validate-express

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+
+require 'nokogiri'
+require 'expressir'
+require 'expressir/express/parser'
+require 'expressir/express/formatter'
+
+stepmod_dir = ARGV.pop || "iso-10303-stepmod-wg12"
+stepmod_path = Pathname.new(stepmod_dir)
+
+SHOULD_REWRITE = false
+
+REPO_INDEX_XML = stepmod_path.join("repository_index.xml")
+
+def all_express_files(stepmod_dir)
+  index_file = File.read(REPO_INDEX_XML)
+  index = Nokogiri::XML(index_file).root
+
+  files = []
+  index.xpath("modules/module").each do |item|
+    files << "#{stepmod_dir}/data/modules/#{item['name']}/arm.exp"
+    files << "#{stepmod_dir}/data/modules/#{item['name']}/mim.exp"
+  end
+
+  index.xpath("resources/resource").each do |item|
+    next if item["name"] == "iso13584_expressions_schema"
+
+    files << "#{stepmod_dir}/data/resources/#{item['name']}/#{item['name']}.exp"
+  end
+
+  index.xpath("business_object_models/business_object_model").each do |item|
+    files << "#{stepmod_dir}/data/business_object_models/#{item['name']}/bom.exp"
+    files << "#{stepmod_dir}/data/business_object_models/#{item['name']}/DomainModel.exp"
+  end
+
+  files.filter { |file| File.exist?(file) }
+end
+
+def parse_express_schema(path)
+  Expressir::Express::Parser.from_file(path)
+rescue
+  puts "ERROR: Broken schema detected in #{path}!"
+  nil
+end
+
+def format_express_schema(repo)
+  Expressir::Express::Formatter.format(repo)
+end
+
+schemas_status = {}
+
+puts "Validating all EXPRESS schema files listed in:"
+puts REPO_INDEX_XML
+puts
+puts "NOTE: SHOULD_REWRITE mode is turned #{SHOULD_REWRITE ? 'ON' : 'OFF'}"
+
+all_express_files(stepmod_dir).each_with_index do |exp_file, i|
+  # puts "i #{i}"
+  # next if i > 10
+
+  puts "validating: #{exp_file}"
+  exp_path = Pathname.new(exp_file)
+  repo = parse_express_schema(exp_file)
+
+
+  schemas_status[exp_path.to_s] = repo ? true : false
+
+  if repo.nil?
+    puts "FAILED    : #{exp_file}"
+    next
+  end
+
+  puts "validated : #{exp_file}"
+
+  # Commented out pretty formatting as some formatted schemas will fail validation...
+  if SHOULD_REWRITE
+    formatted_schema = format_express_schema(repo)
+    # experimental_path = exp_path.dirname.join(exp_path.basename('.exp').to_s+".x.exp")
+
+    puts "rewriting : #{exp_path.to_s}"
+    exp_path.truncate(0)
+    exp_path.write(formatted_schema)
+  end
+
+end
+
+puts "*"*30
+
+if schemas_status.values.include?(false)
+  puts "The following schemas failed validation:"
+  schemas_status.select do |path, status|
+    status == false
+  end.each_pair do |path, status|
+    puts "SCHEMA: #{path}"
+  end
+  puts "*"*30
+
+  exit 1
+end
+
+puts "All schemas passed validation."
+puts "*"*30
+
+exit 0

data/lib/stepmod/utils/stepmod_file_annotator.rb

@@ -130,7 +130,9 @@ module Stepmod
       private
 
       def sanitize(file_content)
-        file_content.gsub("(*)", "(`*`)")
+        file_content
+          .gsub("(*)", "(`*`)")
+          .gsub(";;", ";")
       end
 
       def add_images_references(description, schema_base_dir, processed_images_cache)
@@ -150,6 +152,8 @@ module Stepmod
           processed_images_cache[new_image_file_path] = true
           @images_references[image_file_path] = new_image_file_path
         end
+
+        @images_references
       end
 
       def convert_from_description_text(descriptions_file, description)

data/lib/stepmod/utils/version.rb

@@ -1,5 +1,5 @@
 module Stepmod
   module Utils
-    VERSION = "0.4.0".freeze
+    VERSION = "0.4.1".freeze
   end
 end

data/stepmod-utils.gemspec

@@ -37,6 +37,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "indefinite_article"
   spec.add_runtime_dependency "ptools"
   spec.add_runtime_dependency "pubid-iso"
+  spec.add_runtime_dependency "octokit"
+  spec.add_runtime_dependency "down"
   spec.add_runtime_dependency "reverse_adoc", ">= 0.3"
   spec.add_runtime_dependency "thor", ">= 0.20"
   spec.add_development_dependency "rubocop"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stepmod-utils
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-01 00:00:00.000000000 Z
+date: 2023-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -94,6 +94,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: octokit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: down
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: reverse_adoc
   requirement: !ruby/object:Gem::Requirement
@@ -176,8 +204,7 @@ executables:
 - stepmod-extract-changes
 - stepmod-extract-concepts
 - stepmod-srl-migrate
-- stepmod-srl-migrate-docs
-- stepmod-srl-migrate-schemas
+- stepmod-validate-express
 extensions: []
 extra_rdoc_files: []
 files:
@@ -201,8 +228,7 @@ files:
 - exe/stepmod-extract-changes
 - exe/stepmod-extract-concepts
 - exe/stepmod-srl-migrate
-- exe/stepmod-srl-migrate-docs
-- exe/stepmod-srl-migrate-schemas
+- exe/stepmod-validate-express
 - lib/stepmod/utils.rb
 - lib/stepmod/utils/bibdata.rb
 - lib/stepmod/utils/change.rb

data/exe/stepmod-srl-migrate-docs

@@ -1,82 +0,0 @@
-#!/usr/bin/env ruby
-
-# encoding: UTF-8
-
-require "fileutils"
-# resolve bin path, ignoring symlinks
-require "pathname"
-bin_file = Pathname.new(__FILE__).realpath
-
-# add self to libpath
-$:.unshift File.expand_path("../../lib", bin_file)
-
-# Fixes https://github.com/rubygems/rubygems/issues/1420
-require "rubygems/specification"
-
-module Gem
-  class Specification
-    def this; self; end
-  end
-end
-
-require "bundler/setup"
-require "optparse"
-
-def log(message)
-  puts "[stepmod-utils] #{message}"
-end
-
-options = {}
-OptionParser.new do |opts|
-  opts.banner = "Usage: #{$0} [options]"
-
-  opts.on(
-    "--stepmod-data-dir STEPMOD_DATA_DIR",
-    String,
-    "Path to STEPmod data directory",
-  ) do |path|
-    options[:stepmod_dir] = path
-  end
-
-  opts.on(
-    "--srl-docs-dir SRL_DOCS_DIR",
-    String,
-    "Path to output srl docs",
-  ) do |path|
-    options[:srl_output_docs_dir] = path
-  end
-
-  opts.on(
-    "-s",
-    "--stepmod2mn-executeable STEPMOD2MN_EXECUTEABLE",
-    String,
-    "Path to STEPMOD2MN_EXECUTEABLE",
-  ) do |path|
-    options[:stepmod2mn_path] = path
-  end
-
-  opts.on_tail("-h", "--help", "Show this message") do
-    puts opts
-    exit
-  end
-end.parse!
-
-stepmod2mn_path = options[:stepmod2mn_path]
-if stepmod2mn_path.nil?
-  raise StandardError.new(
-    "stepmod2mn executeable is required, set with the `-s` option.",
-  )
-else
-  log "stepmod2mn executeable path: `#{stepmod2mn_path}`"
-end
-
-stepmod_dir = options[:stepmod_dir] || "stepmod/data"
-log "STEPmod data path: `#{stepmod_dir}`"
-
-srl_output_docs_dir = options[:srl_output_docs_dir] || "documents"
-unless File.directory?(srl_output_docs_dir)
-  FileUtils.mkdir_p(srl_output_docs_dir)
-end
-log "SRL output docs directory path: `#{srl_output_docs_dir}`"
-
-system "java -Xss5m -jar #{stepmod2mn_path} #{stepmod_dir} --output #{srl_output_docs_dir}"

data/exe/stepmod-srl-migrate-schemas

@@ -1,114 +0,0 @@
-#!/usr/bin/env ruby
-
-# encoding: UTF-8
-
-require "fileutils"
-# resolve bin path, ignoring symlinks
-require "pathname"
-bin_file = Pathname.new(__FILE__).realpath
-
-# add self to libpath
-$:.unshift File.expand_path("../../lib", bin_file)
-
-# Fixes https://github.com/rubygems/rubygems/issues/1420
-require "rubygems/specification"
-
-module Gem
-  class Specification
-    def this; self; end
-  end
-end
-
-require "bundler/setup"
-require "optparse"
-
-def log(message, indent = 0)
-  indent_spaces = " " * (indent * 2)
-  puts "[stepmod-utils] #{indent_spaces}#{message}"
-end
-
-def copy_files_to_schemas(path, output_dir)
-  schema_name = path.dirname.basename
-  file_name = path.basename
-
-  new_dir = File.join(output_dir, schema_name)
-  new_file_path = File.join(new_dir, file_name.to_s.gsub("_annotated.exp", ".exp"))
-
-  unless Dir.exist?(new_dir)
-    FileUtils.mkdir_p(new_dir)
-  end
-
-  FileUtils.copy_file(path, new_file_path)
-  log("Copied #{path.basename} to #{new_file_path}", 1)
-end
-
-options = {}
-OptionParser.new do |opts|
-  opts.banner = "Usage: #{$0} [options]"
-
-  opts.on(
-    "--stepmod-data-dir STEPMOD_DATA_DIR",
-    String,
-    "Path to STEPmod data directory",
-  ) do |path|
-    options[:stepmod_dir] = path
-  end
-
-  opts.on(
-    "--srl-schemas-dir SRL_SCHEMAS_DIR",
-    String,
-    "Path to output srl schemas",
-  ) do |path|
-    options[:srl_output_schemas_dir] = path
-  end
-
-  opts.on(
-    "-s",
-    "--stepmod2mn-executeable STEPMOD2MN_EXECUTEABLE",
-    String,
-    "Path to STEPMOD2MN_EXECUTEABLE",
-  ) do |path|
-    options[:stepmod2mn_path] = path
-  end
-
-  opts.on_tail("-h", "--help", "Show this message") do
-    puts opts
-    exit
-  end
-end.parse!
-
-stepmod2mn_path = options[:stepmod2mn_path]
-if stepmod2mn_path.nil?
-  raise StandardError.new(
-    "stepmod2mn executeable is required, set with the `-s` option.",
-  )
-else
-  log "stepmod2mn executeable path: `#{stepmod2mn_path}`"
-end
-
-stepmod_dir = options[:stepmod_dir] || "stepmod/data"
-log "STEPmod data path: `#{stepmod_dir}`"
-
-srl_output_schemas_dir = options[:srl_output_schemas_dir] || "schemas"
-unless File.directory?(srl_output_schemas_dir)
-  FileUtils.mkdir_p(srl_output_schemas_dir)
-end
-log "SRL output schemas directory path: `#{srl_output_schemas_dir}`"
-
-system "java -Xss5m -jar #{stepmod2mn_path} #{stepmod_dir} --output #{srl_output_schemas_dir} --svg"
-
-system "stepmod-annotate-all #{stepmod_dir}"
-
-Dir.glob("#{stepmod_dir}/data/**/*_annotated.exp").each do |filepath|
-  path = Pathname.new(filepath)
-  copy_files_to_schemas(path, srl_output_schemas_dir)
-end
-
-system "stepmod-extract-changes -p #{stepmod_dir}"
-
-Dir.glob("#{stepmod_dir}/data/**/*.changes.yaml").each do |filepath|
-  path = Pathname.new(filepath)
-  copy_files_to_schemas(path, srl_output_schemas_dir)
-end
-
-log("Done")