stepmod-utils 0.3.24 → 0.3.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92d04b3c30620d813fbff40cfa84ed809f0129440dd1f6be45c4206aaa44a12e
-  data.tar.gz: 54289d1067ccba3a61cd0ca07bd5455081efe70aae946811cf8028f6e77afb9a
+  metadata.gz: 46ea2364546e11b067f0d3d21e78d0a1df701ec5189aeb94d08461d551b22786
+  data.tar.gz: ab5adf48414ca093c86ff2fbe12155ff7eab79102412f1c1f9949ee5e7eac591
 SHA512:
-  metadata.gz: e83145314175c1797f8de0fe08bdbf3ba95e06346b49f80beb3cdadb31d1b2423cb60230d869b91850de3685b0fb6c805a2b4ddebbfd72ff2b55ffde234f4d9f
-  data.tar.gz: 309a341bead6e54225f14101c3bea989f2f7873409c4d8e1fedbc6e8b2144d3e21ed2034c5ebcfd76279d73234a84e8d0d67c86fb77e1e1b715f6f1ba2ad661b
+  metadata.gz: 16eb837cff97033e742a20f2513ca426f69d41bb49cce9956c450f41c90290bd3404083c378a40b0ac6c35a68c974c57db1b11e5919cc6a1d5da9b65845b310b
+  data.tar.gz: c3df83672ebda912b592c487698b61e1e19e91cc31da78959dae7d3f4d3836b6a1940d78e33eec34dca92815fe7385e0289f3170303f989187132e1ef21ebc82

data/.github/workflows/test-concept-generation.yml

@@ -0,0 +1,38 @@
+name: test-concept-generation
+
+on:
+  push:
+    branches: [ main ]
+    tags: [ v* ]
+  pull_request:
+
+jobs:
+  generate_yaml_concepts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.1'
+          bundler-cache: true
+
+      - name: Check out iso-10303-stepmod-wg12 repo
+        uses: actions/checkout@v3
+        with:
+          repository: metanorma/iso-10303-stepmod-wg12
+          token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}
+          path: iso-10303-stepmod-wg12
+
+      - name: Generate Annotated EXPRESS files
+        run: bundle exec stepmod-annotate-all ./iso-10303-stepmod-wg12
+
+      - name: Generate Concept YAML files
+        working-directory: iso-10303-stepmod-wg12
+        run: |
+          bundle exec stepmod-extract-concepts \
+            -p ./data \
+            -i ./repository_index.xml \
+            -o ./output-yaml

data/.gitignore

@@ -9,4 +9,7 @@
 
 # rspec failure tracking
 .rspec_status
+.rubocop*
 Gemfile.lock
+
+.rubocop-https*

data/exe/stepmod-annotate-all

@@ -5,14 +5,28 @@ require "stepmod/utils/stepmod_file_annotator"
 
 stepmod_dir = ARGV.first || Dir.pwd
 
-# build resource_docs cache
-resource_docs_cache_file = `mktemp`
-`"#{File.join(__dir__,
-              "stepmod-build-resource-docs-cache")}" "#{stepmod_dir}" > "#{resource_docs_cache_file}"`
-
-# annotate each file
-files = `"#{File.join(__dir__,
-                      "stepmod-find-express-files")}" "#{stepmod_dir}"`.strip.split("\n")
+def all_express_files(stepmod_dir)
+  index_file = File.read(File.join(stepmod_dir, "repository_index.xml"))
+  index = Nokogiri::XML(index_file).root
+
+  files = []
+  index.xpath("modules/module").each do |item|
+    files << "#{stepmod_dir}/data/modules/#{item['name']}/arm.exp"
+    files << "#{stepmod_dir}/data/modules/#{item['name']}/mim.exp"
+  end
+
+  index.xpath("resources/resource").each do |item|
+    files << "#{stepmod_dir}/data/resources/#{item['name']}/#{item['name']}.exp"
+  end
+
+  index.xpath("business_object_models/business_object_model").each do |item|
+    files << "#{stepmod_dir}/data/business_object_models/#{item['name']}/bom.exp"
+    files << "#{stepmod_dir}/data/business_object_models/#{item['name']}/DomainModel.exp"
+  end
+
+  files.filter { |file| File.exist?(file) }
+end
+
 MAX_THREADS = 1 #[2, Concurrent.processor_count].max * 2
 MAX_QUEUE_SIZE = MAX_THREADS * 4
 # https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/thread_pools.md
@@ -22,14 +36,16 @@ pool = Concurrent::ThreadPoolExecutor.new(
   max_queue: MAX_QUEUE_SIZE,
   fallback_policy: :caller_runs,
 )
+
+files = all_express_files(stepmod_dir)
+
 files.each_slice(MAX_QUEUE_SIZE) do |batch|
   puts("Queueing next batch")
   batch.each do |file|
     pool.post do
-      puts("#{Thread.current.object_id}: Queued processing #{file}")
+      puts "#{Thread.current.object_id}: Queued processing #{file}"
       annotated = Stepmod::Utils::StepmodFileAnnotator.new(
         express_file: file,
-        resource_docs_cache_file: resource_docs_cache_file,
         stepmod_dir: stepmod_dir
       ).call
 
@@ -40,11 +56,8 @@ files.each_slice(MAX_QUEUE_SIZE) do |batch|
         file.puts(annotated)
       end
 
-      puts("#{Thread.current.object_id}: Done processing #{File.basename(file)} => #{annotated_file_path}.")
+      puts "#{Thread.current.object_id}: Done processing #{File.basename(file)} => #{annotated_file_path}."
     end
   end
   pool.shutdown
 end
-
-# cleanup
-`rm "#{resource_docs_cache_file}"`

data/exe/stepmod-extract-changes

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+# resolve bin path, ignoring symlinks
+require "pathname"
+bin_file = Pathname.new(__FILE__).realpath
+
+# add self to libpath
+$:.unshift File.expand_path("../../lib", bin_file)
+
+# Fixes https://github.com/rubygems/rubygems/issues/1420
+require "rubygems/specification"
+
+module Gem
+  class Specification
+    def this; self; end
+  end
+end
+
+require "bundler/setup"
+require "stepmod/utils/changes_extractor"
+require "optparse"
+
+def log(message)
+  puts "[stepmod-utils] #{message}"
+end
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.on(
+    "-p",
+    "--path STEPMOD_DATA_PATH",
+    String,
+    "Path to STEPmod data directory",
+  ) do |path|
+    options[:stepmod_dir] = path
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+stepmod_dir = options[:stepmod_dir]
+if stepmod_dir.nil?
+  raise StandardError.new("STEPmod data path not set, set with the `-p` option")
+else
+  log "STEPmod data path: `#{stepmod_dir}`"
+end
+
+changes = Stepmod::Utils::ChangesExtractor.call(
+  stepmod_dir: Pathname.new(stepmod_dir).realpath,
+)
+
+changes.save_to_files

data/exe/{stepmod-extract-yaml-terms → stepmod-extract-concepts}

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # encoding: UTF-8
 
-require "pry"
+require "fileutils"
 # resolve bin path, ignoring symlinks
 require "pathname"
 bin_file = Pathname.new(__FILE__).realpath
@@ -50,6 +50,17 @@ OptionParser.new do |opts|
     end
   end
 
+  opts.on(
+    "-o",
+    "--output INDEX_PATH",
+    String,
+    "Path to output directory",
+  ) do |path|
+    unless path.nil?
+      options[:output_dir] = Pathname.new(path).to_s
+    end
+  end
+
   opts.on_tail("-h", "--help", "Show this message") do
     puts opts
     exit
@@ -67,7 +78,7 @@ end
 
 default_index_path = File.join(stepmod_dir, "repository_index.xml")
 index_path = options[:index_path] || default_index_path
-if File.exists?(index_path)
+if File.exist?(index_path)
   log "Repository index path: `#{index_path}`"
 else
   raise StandardError.new(
@@ -75,6 +86,13 @@ else
   )
 end
 
+default_output_dir = File.join(stepmod_dir, "output_yaml")
+output_dir = options[:output_dir] || default_output_dir
+unless File.directory?(output_dir)
+  FileUtils.mkdir_p(output_dir)
+end
+log "Output directory path: `#{output_dir}`"
+
 _general_concepts,
 resource_concepts,
 _parsed_bibliography,
@@ -122,28 +140,26 @@ def replace_content(content)
 end
 
 part_resources.each do |(_bibdata, current_part_resources)|
-  current_part_resources.save_to_files("./")
+  current_part_resources.save_to_files(output_dir)
 end
 log "INFO: part_resources written to yaml files"
 
-
-log "INFO: written summary file to: 04x-stepmod-entities-resources.adoc"
 part_modules.sort_by do |(bibdata, _part_modules_arm, _part_modules_mim)|
   bibdata.part.to_i
 end.each do |(_bibdata, part_modules_arm, part_modules_mim)|
   unless part_modules_arm.empty?
     part_modules_arm.values.map do |managed_concept|
-      managed_concept.save_to_files("./")
+      managed_concept.save_to_files(output_dir)
     end
   end
 
   unless part_modules_mim.empty?
     part_modules_mim.values.map do |managed_concept|
-      managed_concept.save_to_files("./")
+      managed_concept.save_to_files(output_dir)
     end
   end
 end
 log "INFO: part_modules written to yaml files"
 
-resource_concepts.save_to_files("./")
+resource_concepts.save_to_files(output_dir)
 log "INFO: resource_concepts written to yaml files"

data/lib/stepmod/utils/change.rb

@@ -0,0 +1,74 @@
+require "psych"
+require "stepmod/utils/change_edition"
+require "stepmod/utils/change_edition_collection"
+
+module Stepmod
+  module Utils
+    class Change
+      attr_accessor :schema_name
+      attr_reader :change_editions
+
+      MODULE_TYPES = {
+        arm: "arm",
+        mim: "mim",
+        arm_longform: "arm_lf",
+        mim_longform: "mim_lf",
+      }.freeze
+
+      def initialize(stepmod_dir:, schema_name:, type:)
+        @stepmod_dir = stepmod_dir
+        @change_editions = Stepmod::Utils::ChangeEditionCollection.new
+        @schema_name = schema_name
+        @type = type
+      end
+
+      def resource?
+        !module?
+      end
+
+      def module?
+        MODULE_TYPES.key?(@type.to_sym) || MODULE_TYPES.value?(@type.to_s)
+      end
+
+      def add_change_edition(change_edition)
+        @change_editions[change_edition[:version]] = change_edition
+      end
+
+      def fetch_change_edition(version)
+        @change_editions[version]
+      end
+      alias_method :[], :fetch_change_edition
+
+      def save_to_file
+        File.write(filepath(@type), Psych.dump(to_h))
+      end
+
+      def to_h
+        {
+          "schema" => schema_name,
+          "change_edition" => change_editions.to_h,
+        }
+      end
+
+      private
+
+      def filepath(type)
+        File.join(
+          @stepmod_dir,
+          "data",
+          base_folder,
+          schema_name,
+          "#{MODULE_TYPES[type.to_sym] || schema_name}.changes.yaml",
+        )
+      end
+
+      def base_folder
+        if resource?
+          "resources"
+        else
+          "modules"
+        end
+      end
+    end
+  end
+end

data/lib/stepmod/utils/change_collection.rb

@@ -0,0 +1,50 @@
+require "stepmod/utils/change"
+
+module Stepmod
+  module Utils
+    class ChangeCollection
+      def initialize(stepmod_dir:)
+        @stepmod_dir = stepmod_dir
+        @changes = {}
+      end
+
+      def fetch_or_initialize(change, type)
+        schema = schema_name(change)
+
+        @changes[schema_identifier(schema, type)] ||= Change.new(
+          type: type,
+          stepmod_dir: @stepmod_dir,
+          schema_name: schema,
+        )
+      end
+
+      def fetch(change, type)
+        schema = schema_name(change)
+        @changes[schema_identifier(schema, type)]
+      end
+
+      def save_to_files
+        @changes.values.each(&:save_to_file)
+      end
+
+      def count
+        @changes.keys.count
+      end
+      alias_method :size, :count
+
+      def each(&block)
+        @changes.values.each(&block)
+      end
+
+      private
+
+      def schema_name(change)
+        change.is_a?(Stepmod::Utils::Change) ? change.schema_name : change
+      end
+
+      def schema_identifier(schema_name, type)
+        "#{schema_name}_#{type}"
+      end
+    end
+  end
+end

data/lib/stepmod/utils/change_edition.rb

@@ -0,0 +1,60 @@
+module Stepmod
+  module Utils
+    class ChangeEdition
+      attr_accessor :version, :description
+      attr_reader :additions, :modifications, :deletions, :mapping
+
+      def initialize(options)
+        @version = options[:version]
+        @description = options[:description]
+        self.additions = options[:additions] || []
+        self.modifications = options[:modifications] || []
+        self.deletions = options[:deletions] || []
+        self.mapping = options[:mapping] || []
+      end
+
+      def additions=(additions)
+        validate_type("additions", additions, Array)
+
+        @additions = additions
+      end
+
+      def modifications=(modifications)
+        validate_type("modifications", modifications, Array)
+
+        @modifications = modifications
+      end
+
+      def deletions=(deletions)
+        validate_type("deletions", deletions, Array)
+
+        @deletions = deletions
+      end
+
+      def mapping=(mapping)
+        validate_type("mapping", mapping, Array)
+
+        @mapping = mapping
+      end
+
+      def to_h
+        {
+          "version" => version,
+          "description" => description,
+          "additions" => additions,
+          "modifications" => modifications,
+          "deletions" => deletions,
+          "mapping" => mapping,
+        }.reject { |_k, v| v.nil? || v.empty? }
+      end
+
+      private
+
+      def validate_type(column, value, type)
+        error = "#{column} must be of type ::#{type}, Got ::#{value.class}"
+
+        raise error unless value.is_a?(type)
+      end
+    end
+  end
+end

data/lib/stepmod/utils/change_edition_collection.rb

@@ -0,0 +1,38 @@
+require "stepmod/utils/change_edition"
+
+module Stepmod
+  module Utils
+    class ChangeEditionCollection
+      def initialize
+        @collection = {}
+      end
+
+      def fetch_or_initialize(version)
+        @collection[version] ||=
+          Stepmod::Utils::ChangeEdition.new(version: version)
+      end
+
+      def to_h
+        @collection.values.map(&:to_h)
+      end
+
+      def []=(version, change_edition)
+        klass = Stepmod::Utils::ChangeEdition
+        @collection[version] = if change_edition.is_a?(klass)
+                                 change_edition
+                               else
+                                 klass.new(change_edition)
+                               end
+      end
+
+      def [](version)
+        @collection[version]
+      end
+
+      def count
+        @collection.values.count
+      end
+      alias_method :size, :count
+    end
+  end
+end

data/lib/stepmod/utils/changes_extractor.rb

@@ -0,0 +1,194 @@
+require "nokogiri"
+require "stepmod/utils/change_collection"
+require "stepmod/utils/smrl_description_converter"
+
+module Stepmod
+  module Utils
+    class ChangesExtractor
+      MODULE_TYPES = %w[arm mim arm_longform mim_longform].freeze
+
+      attr_accessor :stepmod_dir
+
+      def self.call(stepmod_dir:, stdout: $stdout)
+        new(stepmod_dir: stepmod_dir, stdout: stdout).call
+      end
+
+      def initialize(stepmod_dir:, stdout: $stdout)
+        @stdout = stdout
+        @stepmod_dir = stepmod_dir
+        @collection = Stepmod::Utils::ChangeCollection.new(
+          stepmod_dir: stepmod_dir,
+        )
+      end
+
+      def call
+        all_resource_changes_files.each do |resource_change_file|
+          xml_changes = Nokogiri::XML(File.read(resource_change_file)).root
+          add_resource_changes_to_collection(xml_changes, @collection)
+        end
+
+        all_modules_changes_files.each do |module_change_file|
+          xml_changes = Nokogiri::XML(File.read(module_change_file)).root
+          schema_name = Pathname.new(module_change_file).parent.basename.to_s
+          add_module_changes_to_collection(xml_changes, @collection, schema_name)
+        end
+
+        @collection
+      end
+
+      private
+
+      # rubocop:disable Metrics/MethodLength
+      def add_resource_changes_to_collection(xml_data, collection)
+        xml_data.xpath("//changes").each do |changes_node|
+          changes_node.xpath("change_edition").each do |change_edition_node|
+            options = {
+              version: change_edition_node.attr("version"),
+              description: Stepmod::Utils::SmrlDescriptionConverter.convert(
+                change_edition_node.at("description"),
+              ),
+            }
+
+            add_resource_changes(
+              collection,
+              change_edition_node,
+              options,
+            )
+          end
+        end
+      end
+
+      def add_module_changes_to_collection(xml_data, collection, schema_name)
+        xml_data.xpath("//changes").each do |changes_node|
+          changes_node.xpath("change").each do |change_node|
+            options = {
+              schema_name: schema_name,
+              version: change_node.attr("version"),
+              description: converted_description(change_node.at("description")),
+            }
+
+            MODULE_TYPES.each do |type|
+              add_module_changes(
+                collection,
+                change_node,
+                options,
+                type,
+              )
+            end
+          end
+        end
+      end
+
+      def add_resource_changes(collection, change_node, options)
+        change_node.xpath("schema.changes").each do |changes|
+          schema_name = correct_schema_name(changes.attr("schema_name"))
+          change = collection.fetch_or_initialize(schema_name, "schema")
+
+          change_edition = extract_change_edition(changes, options)
+          change.add_change_edition(change_edition)
+        end
+      end
+
+      def add_module_changes(collection, change_node, options, type)
+        options[:type] = type
+        # assuming there will only be one change related to a type in the
+        # same version.
+        changes = change_node.xpath("#{type}.changes").first
+        schema_name = options[:schema_name]
+        change = collection.fetch_or_initialize(schema_name, type)
+
+        change_edition = extract_change_edition(changes, options)
+        change.add_change_edition(change_edition)
+
+        if type == "arm"
+          add_mapping_changes(collection, change_node, options)
+        end
+      end
+
+      def add_mapping_changes(collection, change_node, options)
+        change_edition = collection
+                         .fetch_or_initialize(options[:schema_name], "arm")
+                         .change_editions
+                         .fetch_or_initialize(options[:version])
+
+        change_edition.mapping = extract_mapping_changes(change_node)
+      end
+
+      def extract_mapping_changes(change_node)
+        mappings = []
+
+        change_node.xpath("mapping.changes").each do |changes|
+          changes.xpath("mapping.change").each do |change|
+            mappings << { "change" => change.text }
+          end
+
+          changes.xpath("description").each do |change|
+            mappings << { "description" => change.text }
+          end
+        end
+
+        mappings
+      end
+
+      def extract_change_edition(schema_changes, options)
+        type = options[:type] || "schema"
+        addition_nodes = schema_changes&.xpath("#{type}.additions") || []
+        modification_nodes = schema_changes&.xpath("#{type}.modifications") || []
+        deletion_nodes = schema_changes&.xpath("#{type}.deletions") || []
+
+        {
+          version: options[:version],
+          description: options[:description],
+          additions: extract_modified_objects(addition_nodes),
+          modifications: extract_modified_objects(modification_nodes),
+          deletions: extract_modified_objects(deletion_nodes),
+        }
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def extract_modified_objects(nodes)
+        nodes.map do |node|
+          node.xpath("modified.object").map do |object|
+            {
+              type: object.attr("type"),
+              name: object.attr("name"),
+              description: converted_description(object.at("description")),
+              interfaced_items: object.attr("interfaced.items"),
+            }.compact
+          end
+        end.flatten
+      end
+
+      def converted_description(description)
+        return if description.to_s.empty?
+
+        Stepmod::Utils::SmrlDescriptionConverter.convert(description)
+      end
+
+      def all_resource_changes_files
+        Dir.glob(
+          File.join(stepmod_dir, "data", "resource_docs", "*", "resource.xml"),
+        )
+      end
+
+      def all_modules_changes_files
+        Dir.glob(
+          File.join(stepmod_dir, "data", "modules", "*", "module.xml"),
+        )
+      end
+
+      # rubocop:disable Layout/LineLength
+      def correct_schema_name(schema_name)
+        schema_name_corrector = {
+          "material_property_definition" => "material_property_definition_schema",
+          "qualified_measure" => "qualified_measure_schema",
+          "material_property_representation" => "material_property_representation_schema",
+          "annotated_3d_model_data_quality_criteria" => "annotated_3d_model_data_quality_criteria_schema",
+        }
+
+        schema_name_corrector[schema_name] || schema_name
+      end
+      # rubocop:enable Layout/LineLength
+    end
+  end
+end

data/lib/stepmod/utils/converters/description.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Stepmod
+  module Utils
+    module Converters
+      class Description < ReverseAdoc::Converters::Base
+        def convert(node, state = {})
+          treat_children(node, state)
+        end
+
+        private
+
+        def treat_children(node, state)
+          res = node.children.map { |child| treat(child, state) }
+          res.map(&:strip).reject(&:empty?).join("\n\n")
+        end
+      end
+
+      ReverseAdoc::Converters.register :description, Description.new
+    end
+  end
+end

data/lib/stepmod/utils/smrl_description_converter.rb

@@ -6,6 +6,7 @@ require "stepmod/utils/converters/blockquote"
 require "stepmod/utils/converters/br"
 require "stepmod/utils/converters/bypass"
 require "stepmod/utils/converters/code"
+require "stepmod/utils/converters/description"
 require "stepmod/utils/converters/drop"
 require "stepmod/utils/converters/em_express_description"
 require "stepmod/utils/converters/example"

data/lib/stepmod/utils/stepmod_file_annotator.rb

@@ -12,13 +12,13 @@ require "pubid-iso"
 module Stepmod
   module Utils
     class StepmodFileAnnotator
-      attr_reader :express_file, :resource_docs_cache_file, :stepmod_dir
+      attr_reader :express_file, :resource_docs_cache, :stepmod_dir
 
       # @param express_file [String] path to the exp file needed to annotate
-      # @param resource_docs_cache_file [String] output of ./stepmod-build-resource-docs-cache
-      def initialize(express_file:, resource_docs_cache_file:, stepmod_dir: nil)
+      # @param resource_docs_cache [String] output of ./stepmod-build-resource-docs-cache
+      def initialize(express_file:, stepmod_dir: nil)
         @express_file = express_file
-        @resource_docs_cache_file = resource_docs_cache_file
+        @resource_docs_cache = resource_docs_schemas(stepmod_dir)
         @stepmod_dir = stepmod_dir || Dir.pwd
         @added_bibdata = {}
 
@@ -28,6 +28,23 @@ module Stepmod
                                                  .id
       end
 
+      def resource_docs_schemas(stepmod_dir)
+        filepath = File.join(stepmod_dir, "data/resource_docs/*/resource.xml")
+
+        schemas = {}
+        Dir.glob(filepath).each do |resource_docs_file|
+          match = resource_docs_file.match("data/resource_docs/([^/]+)/resource.xml")
+          resource_docs_dir = match.captures[0]
+
+          resource_docs = Nokogiri::XML(File.read(resource_docs_file)).root
+          resource_docs.xpath("schema").each do |schema|
+            schemas[schema["name"]] = resource_docs_dir
+          end
+        end
+
+        schemas
+      end
+
       def call
         match = File.basename(express_file).match('^(arm|mim|bom)\.exp$')
         descriptions_base = match ? "#{match.captures[0]}_descriptions.xml" : "descriptions.xml"
@@ -39,7 +56,7 @@ module Stepmod
         converted_description = ""
         base_linked = ""
 
-        if File.exists?(descriptions_file)
+        if File.exist?(descriptions_file)
           descriptions = Nokogiri::XML(File.read(descriptions_file)).root
           added_resource_descriptions = {}
 
@@ -81,7 +98,7 @@ module Stepmod
                      resource_docs_file_path(stepmod_dir, bib_file_name)
                    end
 
-        output_express << if bib_file && File.exists?(bib_file)
+        output_express << if bib_file && File.exist?(bib_file)
                             prepend_bibdata(
                               converted_description || "",
                               # bib_file will not be present for resouces
@@ -107,10 +124,6 @@ module Stepmod
         file_content.gsub("(*)", "(`*`)")
       end
 
-      def resource_docs_cache
-        @resource_docs_cache ||= JSON.parse(File.read(resource_docs_cache_file))
-      end
-
       def convert_from_description_text(descriptions_file, description)
         Dir.chdir(File.dirname(descriptions_file)) do
           wrapper = "<ext_descriptions>#{description}</ext_descriptions>"

data/lib/stepmod/utils/terms_extractor.rb

@@ -16,6 +16,7 @@ module Stepmod
       # TODO: we may want a command line option to override this in the future
       ACCEPTED_STAGES = %w(IS DIS FDIS TS).freeze
       WITHDRAWN_STATUS = "withdrawn".freeze
+      REDUNDENT_NOTE_REGEX = /^An? .*? is a type of \{\{[^}]*\}\}\s*?\.?$/.freeze
 
       attr_reader :stepmod_path,
                   :stepmod_dir,
@@ -86,8 +87,8 @@ module Stepmod
           arm_path = Pathname.new("#{stepmod_dir}/modules/#{x['name']}/arm_annotated.exp")
           mim_path = Pathname.new("#{stepmod_dir}/modules/#{x['name']}/mim_annotated.exp")
 
-          files << arm_path if File.exists? arm_path
-          files << mim_path if File.exists? mim_path
+          files << arm_path if File.exist? arm_path
+          files << mim_path if File.exist? mim_path
         end
 
         # Should ignore these because the `<resource_docs>` elements do not provide any EXPRESS schemas
@@ -104,7 +105,7 @@ module Stepmod
           next if x['status'] == WITHDRAWN_STATUS
 
           path = Pathname.new("#{stepmod_dir}/resources/#{x['name']}/#{x['name']}_annotated.exp")
-          files << path if File.exists? path
+          files << path if File.exist? path
         end
 
         # Should ignore these because we are skiping Clause 3 terms
@@ -318,6 +319,8 @@ module Stepmod
         old_definition = trim_definition(entity.remarks.first)
         definition = generate_entity_definition(entity, domain)
 
+        notes = [old_definition].reject { |note| redundant_note?(note) }
+
         Stepmod::Utils::Concept.new(
           designations: [
             {
@@ -336,7 +339,7 @@ module Stepmod
               "link" => "https://www.iso.org/standard/32858.html",
             },
           ],
-          notes: [old_definition].compact,
+          notes: notes,
           language_code: "en",
           part: bibdata.part,
           schema: schema,
@@ -512,6 +515,10 @@ module Stepmod
           REMARK
         end.join
       end
+
+      def redundant_note?(note)
+        note && note.match?(REDUNDENT_NOTE_REGEX) && !note.include?("\n")
+      end
     end
   end
 end

data/lib/stepmod/utils/version.rb

@@ -1,5 +1,5 @@
 module Stepmod
   module Utils
-    VERSION = "0.3.24".freeze
+    VERSION = "0.3.25".freeze
   end
 end

data/stepmod-utils.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "concurrent-ruby"
   spec.add_runtime_dependency "expressir"
-  spec.add_runtime_dependency "glossarist-new", "~> 1.0.1"
+  spec.add_runtime_dependency "glossarist", "~> 1.0.5"
   spec.add_runtime_dependency "indefinite_article"
   spec.add_runtime_dependency "ptools"
   spec.add_runtime_dependency "pubid-iso"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stepmod-utils
 version: !ruby/object:Gem::Version
-  version: 0.3.24
+  version: 0.3.25
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-11-24 00:00:00.000000000 Z
+date: 2023-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -39,19 +39,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: glossarist-new
+  name: glossarist
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: 1.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: 1.0.5
 - !ruby/object:Gem::Dependency
   name: indefinite_article
   requirement: !ruby/object:Gem::Requirement
@@ -171,17 +171,16 @@ email:
 - open.source@ribose.com
 executables:
 - stepmod-annotate-all
-- stepmod-build-resource-docs-cache
 - stepmod-convert-express-description
 - stepmod-convert-express-resource
-- stepmod-extract-terms
-- stepmod-extract-yaml-terms
-- stepmod-find-express-files
+- stepmod-extract-changes
+- stepmod-extract-concepts
 extensions: []
 extra_rdoc_files: []
 files:
 - ".github/workflows/rake.yml"
 - ".github/workflows/release.yml"
+- ".github/workflows/test-concept-generation.yml"
 - ".gitignore"
 - ".hound.yml"
 - ".rspec"
@@ -194,14 +193,17 @@ files:
 - bin/console
 - bin/setup
 - exe/stepmod-annotate-all
-- exe/stepmod-build-resource-docs-cache
 - exe/stepmod-convert-express-description
 - exe/stepmod-convert-express-resource
-- exe/stepmod-extract-terms
-- exe/stepmod-extract-yaml-terms
-- exe/stepmod-find-express-files
+- exe/stepmod-extract-changes
+- exe/stepmod-extract-concepts
 - lib/stepmod/utils.rb
 - lib/stepmod/utils/bibdata.rb
+- lib/stepmod/utils/change.rb
+- lib/stepmod/utils/change_collection.rb
+- lib/stepmod/utils/change_edition.rb
+- lib/stepmod/utils/change_edition_collection.rb
+- lib/stepmod/utils/changes_extractor.rb
 - lib/stepmod/utils/cleaner.rb
 - lib/stepmod/utils/concept.rb
 - lib/stepmod/utils/converters/a.rb
@@ -215,6 +217,7 @@ files:
 - lib/stepmod/utils/converters/dd.rb
 - lib/stepmod/utils/converters/def.rb
 - lib/stepmod/utils/converters/definition.rb
+- lib/stepmod/utils/converters/description.rb
 - lib/stepmod/utils/converters/dl.rb
 - lib/stepmod/utils/converters/drop.rb
 - lib/stepmod/utils/converters/dt.rb
@@ -287,7 +290,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
 summary: Stepmod-utils is a toolkit that works on STEPmod data.

data/exe/stepmod-build-resource-docs-cache

@@ -1,20 +0,0 @@
-#!/usr/bin/env ruby
-
-require "nokogiri"
-require "json"
-
-stepmod_dir = ARGV.first || Dir.pwd
-
-schemas = {}
-Dir.glob(File.join(stepmod_dir,
-                   "data/resource_docs/*/resource.xml")).each do |resource_docs_file|
-  match = resource_docs_file.match("data/resource_docs/([^/]+)/resource.xml")
-  resource_docs_dir = match.captures[0]
-
-  resource_docs = Nokogiri::XML(File.read(resource_docs_file)).root
-  resource_docs.xpath("schema").each do |schema|
-    schemas[schema["name"]] = resource_docs_dir
-  end
-end
-
-puts JSON.pretty_generate(schemas)

data/exe/stepmod-extract-terms

@@ -1,237 +0,0 @@
-#!/usr/bin/env ruby
-# encoding: UTF-8
-
-require "pry"
-# resolve bin path, ignoring symlinks
-require "pathname"
-bin_file = Pathname.new(__FILE__).realpath
-
-# add self to libpath
-$:.unshift File.expand_path("../../lib", bin_file)
-
-# Fixes https://github.com/rubygems/rubygems/issues/1420
-require "rubygems/specification"
-
-module Gem
-  class Specification
-    def this; self; end
-  end
-end
-
-require "bundler/setup"
-require "stepmod/utils/terms_extractor"
-require 'optparse'
-
-def log(message)
-  puts "[stepmod-utils] #{message}"
-end
-
-options = {}
-OptionParser.new do |opts|
-  opts.banner = "Usage: #{$0} [options]"
-
-  opts.on("-p", "--path STEPMOD_DATA_PATH", String, "Path to STEPmod data directory") do |path|
-    options[:stepmod_dir] = path
-  end
-
-  opts.on("-i", "--index INDEX_PATH", String, "Path to repository_index.xml") do |path|
-    unless path.nil?
-      options[:index_path] = Pathname.new(path).to_s
-    end
-  end
-
-  opts.on_tail("-h", "--help", "Show this message") do
-    puts opts
-    exit
-  end
-end.parse!
-
-stepmod_dir = options[:stepmod_dir]
-if stepmod_dir.nil?
-  raise StandardError.new("STEPmod data path not set, set with the `-p` option.")
-else
-  log "STEPmod data path: `#{stepmod_dir}`"
-end
-
-index_path = options[:index_path] || File.join(stepmod_dir, "repository_index.xml")
-unless File.exists?(index_path)
-  raise StandardError.new("Index file not present at #{index_path}, set with the `-i` option.")
-else
-  log "Repository index path: `#{index_path}`"
-end
-
-general_concepts,
-resource_concepts,
-parsed_bibliography,
-part_concepts,
-part_resources,
-part_modules = Stepmod::Utils::TermsExtractor.call(stepmod_dir, index_path)
-
-def part_to_title(bibdata)
-  case bibdata.part.to_i
-  when 41
-    "Fundamentals of product description and support"
-  when 42
-    "Geometric and topological representation"
-  when 43
-    "Foundation representation"
-  when 44
-    "Product structure, concept and configuration"
-  when 45
-    "Material and other engineering properties"
-  when 46
-    "Visual presentation"
-  when 47
-    "Shape tolerance"
-  when 51
-    "Mathematical representation"
-  else
-    bibdata.title_en
-  end
-end
-
-IMAGE_REPLACEMENTS = {
-  'image::eq01.gif[]' => 'stem:[H(A,B) = max {h(A, B), h(B,A)}]',
-  'image::eq02.gif[]' => 'stem:[max_{a in A} { min_{b in B} d(a,b) }]',
-  'image::vector_z_c.gif[]' => 'stem:[bar z_{c}]',
-  'image::one_direction_repeat_factor_expression.gif[]' => 'stem:[I + k cdot R; k = -1, 1]',
-  'image::two_direction_repeat_factor_expression.gif[]' => 'stem:[I + k_1 cdot R_1 + k_2 cdot R_2; k_1, k_2 = -1, 0, 1, k^2_1 + k^2_2 != 0]',
-}
-
-TEXT_REPLACEMENTS = {
-  ' (see <module_ref linkend="ply_orientation_specification:4_entities:figure:f2"> Figure 2</module_ref>)' => '',
-  ' (see <module_ref linkend="ply_orientation_specification:4_entities:figure:f3"> Figure 3</module_ref>)' => ''
-}
-
-def replace_content(content)
-  IMAGE_REPLACEMENTS.each_pair do |k, v|
-    content.gsub!(k, v)
-  end
-  TEXT_REPLACEMENTS.each_pair do |k, v|
-    content.gsub!(k, v)
-  end
-
-  content
-end
-
-## Skip all Clause 3 terms
-# part_concepts.each do |(bibdata, current_part_concepts)|
-#   current_part_concepts = current_part_concepts.to_a.map do |n|
-#     n.localizations["en"]
-#   end
-#   fn = "03x-stepmod-#{bibdata.part}.adoc"
-#   File.open(fn, "w") do |file|
-#     file.puts("== #{part_to_title(bibdata)}\n\n")
-#     file.puts(replace_images(current_part_concepts.map(&:to_mn_adoc).join("\n")))
-#   end
-#   log "INFO: written to: #{fn}"
-# end
-
-# File.open("03x-stepmod.adoc", "w") do |file|
-#   part_concepts.sort_by do |(bibdata, current_part_concepts)|
-#     bibdata.part.to_i
-#   end.each do |(bibdata, current_part_concepts)|
-#     fn = "03x-stepmod-#{bibdata.part}.adoc"
-#     file.puts("\ninclude::#{fn}[]\n")
-#   end
-# end
-
-# log "INFO: written summary file to: 03x-stepmod.adoc"
-
-part_resources.each do |(bibdata, current_part_resources)|
-  current_part_resources = current_part_resources.to_a.map do |n|
-    n.localizations["en"]
-  end
-  fn = "04x-stepmod-entities-resources-#{bibdata.part}.adoc"
-  File.open(fn, "w") do |file|
-    # file.puts("== #{part_to_title(bibdata)}\n\n")
-    file.puts(replace_content(current_part_resources.map(&:to_mn_adoc).join("\n")))
-  end
-  log "INFO: written to: #{fn}"
-end
-
-File.open("04x-stepmod-entities-resources.adoc", "w") do |file|
-  part_resources.sort_by do |(bibdata, current_part_resources)|
-    bibdata.part.to_i
-  end.each do |(bibdata, current_part_resources)|
-    fn = "04x-stepmod-entities-resources-#{bibdata.part}.adoc"
-    file.puts("\ninclude::#{fn}[]\n")
-  end
-end
-
-log "INFO: written summary file to: 04x-stepmod-entities-resources.adoc"
-
-part_modules.sort_by do |(bibdata, part_modules_arm, part_modules_mim)|
-  bibdata.part.to_i
-end.each do |(bibdata, part_modules_arm, part_modules_mim)|
-  fn = "05x-stepmod-entities-modules-#{bibdata.part}.adoc"
-  File.open(fn, "w") do |file|
-    file.puts("")
-
-    unless part_modules_arm.empty?
-      schema_name = part_modules_arm.first.first
-      concepts = part_modules_arm.first.last.to_a.map do |n|
-        n.localizations["en"]
-      end
-
-      # puts "SCHEMA NAME ARM: #{schema_name}"
-      file.puts("== #{schema_name}\n\n")
-      file.puts(concepts.map(&:to_mn_adoc).join("\n"))
-    end
-
-    file.puts("")
-
-    unless part_modules_mim.empty?
-      schema_name = part_modules_mim.first.first
-
-      # puts "SCHEMA NAME MIM: #{schema_name}"
-      concepts = part_modules_mim.first.last.to_a.map do |n|
-        n.localizations["en"]
-      end
-      file.puts("== #{schema_name}\n\n")
-      file.puts(replace_content(concepts.map(&:to_mn_adoc).join("\n")))
-    end
-  end
-
-  log "INFO: written to: #{fn}"
-end
-
-File.open("05x-stepmod-entities-modules.adoc", "w") do |file|
-  part_modules.sort_by do |(bibdata, part_modules_arm, part_modules_mim)|
-    bibdata.part.to_i
-  end.each do |(bibdata, part_modules_arm, part_modules_mim)|
-    fn = "05x-stepmod-entities-modules-#{bibdata.part}.adoc"
-    file.puts("\ninclude::#{fn}[]\n")
-  end
-end
-
-log "INFO: written summary file to: 05x-stepmod-entities-modules.adoc"
-
-## Skipping all Clause 3 terms
-# File.open("031-stepmod-general.adoc", "w") do |file|
-#   file.puts(
-#     replace_images(
-#       general_concepts.to_a.map do |n|
-#         n.localizations["en"]
-#       end.map(&:to_mn_adoc).join("\n")
-#     )
-#   )
-# end
-
-# log "INFO: written to: 031-stepmod-general.adoc"
-
-File.open("041-stepmod-entities-resources.adoc", "w") do |file|
-  file.puts(
-    replace_content(
-      resource_concepts.to_a.map do |n|
-        n.localizations["en"]
-      end.map(&:to_mn_adoc).join("\n")
-    )
-  )
-end
-log "INFO: written to: 041-stepmod-entities-resources.adoc"
-
-File.open("991-generated-bibliography.adoc", "w") do |file|
-  file.puts(parsed_bibliography.map(&:to_mn_adoc).sort.uniq.join("\n"))
-end
-log "INFO: written to: 991-generated-bibliography.adoc"

data/exe/stepmod-find-express-files

@@ -1,24 +0,0 @@
-#!/usr/bin/env ruby
-
-require "nokogiri"
-
-stepmod_dir = ARGV.first || Dir.pwd
-
-index = Nokogiri::XML(File.read(File.join(stepmod_dir,
-                                          "repository_index.xml"))).root
-
-files = []
-index.xpath("modules/module").each do |item|
-  files << "#{stepmod_dir}/data/modules/#{item['name']}/arm.exp"
-  files << "#{stepmod_dir}/data/modules/#{item['name']}/mim.exp"
-end
-index.xpath("resources/resource").each do |item|
-  files << "#{stepmod_dir}/data/resources/#{item['name']}/#{item['name']}.exp"
-end
-index.xpath("business_object_models/business_object_model").each do |item|
-  files << "#{stepmod_dir}/data/business_object_models/#{item['name']}/bom.exp"
-  files << "#{stepmod_dir}/data/business_object_models/#{item['name']}/DomainModel.exp"
-end
-
-existing_files = files.filter { |file| File.exists?(file) }
-puts existing_files