stepmod-utils 0.3.23 → 0.3.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96740d6528de476007a4f312fa8889887e77a27d4a693924531813188a3cd60b
-  data.tar.gz: 867240182ffe77c721fe7ffa8ee53b77383db9954cffa9c4de723e652ec90780
+  metadata.gz: 46ea2364546e11b067f0d3d21e78d0a1df701ec5189aeb94d08461d551b22786
+  data.tar.gz: ab5adf48414ca093c86ff2fbe12155ff7eab79102412f1c1f9949ee5e7eac591
 SHA512:
-  metadata.gz: 6e5b71123d326d72e7f6f64269238e2c0db08cec4033794a1a104088d8ae3c0b7e9b005b2551bb2541b06d676fb6959120a7bd1aac30c3cc63068e581206d8c3
-  data.tar.gz: 9aba582cd94d7c7b85fd9bc8ad2e718c0bf23f6609a86de46393c55a85bda496fecbe9f8c847fd073a5aa1ca351148917520a490d695769fde21d118428fbe98
+  metadata.gz: 16eb837cff97033e742a20f2513ca426f69d41bb49cce9956c450f41c90290bd3404083c378a40b0ac6c35a68c974c57db1b11e5919cc6a1d5da9b65845b310b
+  data.tar.gz: c3df83672ebda912b592c487698b61e1e19e91cc31da78959dae7d3f4d3836b6a1940d78e33eec34dca92815fe7385e0289f3170303f989187132e1ef21ebc82

data/.github/workflows/test-concept-generation.yml

@@ -0,0 +1,38 @@
+name: test-concept-generation
+
+on:
+  push:
+    branches: [ main ]
+    tags: [ v* ]
+  pull_request:
+
+jobs:
+  generate_yaml_concepts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.1'
+          bundler-cache: true
+
+      - name: Check out iso-10303-stepmod-wg12 repo
+        uses: actions/checkout@v3
+        with:
+          repository: metanorma/iso-10303-stepmod-wg12
+          token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}
+          path: iso-10303-stepmod-wg12
+
+      - name: Generate Annotated EXPRESS files
+        run: bundle exec stepmod-annotate-all ./iso-10303-stepmod-wg12
+
+      - name: Generate Concept YAML files
+        working-directory: iso-10303-stepmod-wg12
+        run: |
+          bundle exec stepmod-extract-concepts \
+            -p ./data \
+            -i ./repository_index.xml \
+            -o ./output-yaml

data/.gitignore

@@ -9,4 +9,7 @@
 
 # rspec failure tracking
 .rspec_status
+.rubocop*
 Gemfile.lock
+
+.rubocop-https*

data/.rubocop.yml

@@ -6,5 +6,8 @@ inherit_from:
 # local repo-specific modifications
 # ...
 
+Style/MultilineMethodCallIndentation:
+  EnforcedStyle: aligned
+
 AllCops:
   TargetRubyVersion: 2.5

data/exe/stepmod-annotate-all

@@ -5,14 +5,28 @@ require "stepmod/utils/stepmod_file_annotator"
 
 stepmod_dir = ARGV.first || Dir.pwd
 
-# build resource_docs cache
-resource_docs_cache_file = `mktemp`
-`"#{File.join(__dir__,
-              "stepmod-build-resource-docs-cache")}" "#{stepmod_dir}" > "#{resource_docs_cache_file}"`
-
-# annotate each file
-files = `"#{File.join(__dir__,
-                      "stepmod-find-express-files")}" "#{stepmod_dir}"`.strip.split("\n")
+def all_express_files(stepmod_dir)
+  index_file = File.read(File.join(stepmod_dir, "repository_index.xml"))
+  index = Nokogiri::XML(index_file).root
+
+  files = []
+  index.xpath("modules/module").each do |item|
+    files << "#{stepmod_dir}/data/modules/#{item['name']}/arm.exp"
+    files << "#{stepmod_dir}/data/modules/#{item['name']}/mim.exp"
+  end
+
+  index.xpath("resources/resource").each do |item|
+    files << "#{stepmod_dir}/data/resources/#{item['name']}/#{item['name']}.exp"
+  end
+
+  index.xpath("business_object_models/business_object_model").each do |item|
+    files << "#{stepmod_dir}/data/business_object_models/#{item['name']}/bom.exp"
+    files << "#{stepmod_dir}/data/business_object_models/#{item['name']}/DomainModel.exp"
+  end
+
+  files.filter { |file| File.exist?(file) }
+end
+
 MAX_THREADS = 1 #[2, Concurrent.processor_count].max * 2
 MAX_QUEUE_SIZE = MAX_THREADS * 4
 # https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/thread_pools.md
@@ -22,14 +36,16 @@ pool = Concurrent::ThreadPoolExecutor.new(
   max_queue: MAX_QUEUE_SIZE,
   fallback_policy: :caller_runs,
 )
+
+files = all_express_files(stepmod_dir)
+
 files.each_slice(MAX_QUEUE_SIZE) do |batch|
   puts("Queueing next batch")
   batch.each do |file|
     pool.post do
-      puts("#{Thread.current.object_id}: Queued processing #{file}")
+      puts "#{Thread.current.object_id}: Queued processing #{file}"
       annotated = Stepmod::Utils::StepmodFileAnnotator.new(
         express_file: file,
-        resource_docs_cache_file: resource_docs_cache_file,
         stepmod_dir: stepmod_dir
       ).call
 
@@ -40,11 +56,8 @@ files.each_slice(MAX_QUEUE_SIZE) do |batch|
         file.puts(annotated)
       end
 
-      puts("#{Thread.current.object_id}: Done processing #{File.basename(file)} => #{annotated_file_path}.")
+      puts "#{Thread.current.object_id}: Done processing #{File.basename(file)} => #{annotated_file_path}."
     end
   end
   pool.shutdown
 end
-
-# cleanup
-`rm "#{resource_docs_cache_file}"`

data/exe/stepmod-extract-changes

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+# resolve bin path, ignoring symlinks
+require "pathname"
+bin_file = Pathname.new(__FILE__).realpath
+
+# add self to libpath
+$:.unshift File.expand_path("../../lib", bin_file)
+
+# Fixes https://github.com/rubygems/rubygems/issues/1420
+require "rubygems/specification"
+
+module Gem
+  class Specification
+    def this; self; end
+  end
+end
+
+require "bundler/setup"
+require "stepmod/utils/changes_extractor"
+require "optparse"
+
+def log(message)
+  puts "[stepmod-utils] #{message}"
+end
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.on(
+    "-p",
+    "--path STEPMOD_DATA_PATH",
+    String,
+    "Path to STEPmod data directory",
+  ) do |path|
+    options[:stepmod_dir] = path
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+stepmod_dir = options[:stepmod_dir]
+if stepmod_dir.nil?
+  raise StandardError.new("STEPmod data path not set, set with the `-p` option")
+else
+  log "STEPmod data path: `#{stepmod_dir}`"
+end
+
+changes = Stepmod::Utils::ChangesExtractor.call(
+  stepmod_dir: Pathname.new(stepmod_dir).realpath,
+)
+
+changes.save_to_files

data/exe/stepmod-extract-concepts

@@ -0,0 +1,165 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require "fileutils"
+# resolve bin path, ignoring symlinks
+require "pathname"
+bin_file = Pathname.new(__FILE__).realpath
+
+# add self to libpath
+$:.unshift File.expand_path("../../lib", bin_file)
+
+# Fixes https://github.com/rubygems/rubygems/issues/1420
+require "rubygems/specification"
+
+module Gem
+  class Specification
+    def this; self; end
+  end
+end
+
+require "bundler/setup"
+require_relative "../lib/stepmod/utils/terms_extractor"
+require "optparse"
+
+def log(message)
+  puts "[stepmod-utils] #{message}"
+end
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.on(
+    "-p",
+    "--path STEPMOD_DATA_PATH",
+    String,
+    "Path to STEPmod data directory",
+  ) do |path|
+    options[:stepmod_dir] = path
+  end
+
+  opts.on(
+    "-i",
+    "--index INDEX_PATH",
+    String,
+    "Path to repository_index.xml",
+  ) do |path|
+    unless path.nil?
+      options[:index_path] = Pathname.new(path).to_s
+    end
+  end
+
+  opts.on(
+    "-o",
+    "--output INDEX_PATH",
+    String,
+    "Path to output directory",
+  ) do |path|
+    unless path.nil?
+      options[:output_dir] = Pathname.new(path).to_s
+    end
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+stepmod_dir = options[:stepmod_dir]
+if stepmod_dir.nil?
+  raise StandardError.new(
+    "STEPmod data path not set, set with the `-p` option.",
+  )
+else
+  log "STEPmod data path: `#{stepmod_dir}`"
+end
+
+default_index_path = File.join(stepmod_dir, "repository_index.xml")
+index_path = options[:index_path] || default_index_path
+if File.exist?(index_path)
+  log "Repository index path: `#{index_path}`"
+else
+  raise StandardError.new(
+    "Index file not present at #{index_path}, set with the `-i` option.",
+  )
+end
+
+default_output_dir = File.join(stepmod_dir, "output_yaml")
+output_dir = options[:output_dir] || default_output_dir
+unless File.directory?(output_dir)
+  FileUtils.mkdir_p(output_dir)
+end
+log "Output directory path: `#{output_dir}`"
+
+_general_concepts,
+resource_concepts,
+_parsed_bibliography,
+_part_concepts,
+part_resources,
+part_modules = Stepmod::Utils::TermsExtractor.call(stepmod_dir, index_path)
+
+def part_to_title(bibdata)
+  {
+    41 => "Fundamentals of product description and support",
+    42 => "Geometric and topological representation",
+    43 => "Foundation representation",
+    44 => "Product structure, concept and configuration",
+    45 => "Material and other engineering properties",
+    46 => "Visual presentation",
+    47 => "Shape tolerance",
+    51 => "Mathematical representation",
+  }[bibdata.part.to_i] || bibdata.title_en
+end
+
+# rubocop:disable Layout/LineLength
+IMAGE_REPLACEMENTS = {
+  "image::eq01.gif[]" => "stem:[H(A,B) = max {h(A, B), h(B,A)}]",
+  "image::eq02.gif[]" => "stem:[max_{a in A} { min_{b in B} d(a,b) }]",
+  "image::vector_z_c.gif[]" => "stem:[bar z_{c}]",
+  "image::one_direction_repeat_factor_expression.gif[]" => "stem:[I + k cdot R; k = -1, 1]",
+  "image::two_direction_repeat_factor_expression.gif[]" => "stem:[I + k_1 cdot R_1 + k_2 cdot R_2; k_1, k_2 = -1, 0, 1, k^2_1 + k^2_2 != 0]",
+}.freeze
+
+TEXT_REPLACEMENTS = {
+  ' (see <module_ref linkend="ply_orientation_specification:4_entities:figure:f2"> Figure 2</module_ref>)' => "",
+  ' (see <module_ref linkend="ply_orientation_specification:4_entities:figure:f3"> Figure 3</module_ref>)' => "",
+}.freeze
+# rubocop:enable Layout/LineLength
+
+def replace_content(content)
+  IMAGE_REPLACEMENTS.each_pair do |k, v|
+    content.gsub!(k, v)
+  end
+  TEXT_REPLACEMENTS.each_pair do |k, v|
+    content.gsub!(k, v)
+  end
+
+  content
+end
+
+part_resources.each do |(_bibdata, current_part_resources)|
+  current_part_resources.save_to_files(output_dir)
+end
+log "INFO: part_resources written to yaml files"
+
+part_modules.sort_by do |(bibdata, _part_modules_arm, _part_modules_mim)|
+  bibdata.part.to_i
+end.each do |(_bibdata, part_modules_arm, part_modules_mim)|
+  unless part_modules_arm.empty?
+    part_modules_arm.values.map do |managed_concept|
+      managed_concept.save_to_files(output_dir)
+    end
+  end
+
+  unless part_modules_mim.empty?
+    part_modules_mim.values.map do |managed_concept|
+      managed_concept.save_to_files(output_dir)
+    end
+  end
+end
+log "INFO: part_modules written to yaml files"
+
+resource_concepts.save_to_files(output_dir)
+log "INFO: resource_concepts written to yaml files"

data/lib/stepmod/utils/change.rb

@@ -0,0 +1,74 @@
+require "psych"
+require "stepmod/utils/change_edition"
+require "stepmod/utils/change_edition_collection"
+
+module Stepmod
+  module Utils
+    class Change
+      attr_accessor :schema_name
+      attr_reader :change_editions
+
+      MODULE_TYPES = {
+        arm: "arm",
+        mim: "mim",
+        arm_longform: "arm_lf",
+        mim_longform: "mim_lf",
+      }.freeze
+
+      def initialize(stepmod_dir:, schema_name:, type:)
+        @stepmod_dir = stepmod_dir
+        @change_editions = Stepmod::Utils::ChangeEditionCollection.new
+        @schema_name = schema_name
+        @type = type
+      end
+
+      def resource?
+        !module?
+      end
+
+      def module?
+        MODULE_TYPES.key?(@type.to_sym) || MODULE_TYPES.value?(@type.to_s)
+      end
+
+      def add_change_edition(change_edition)
+        @change_editions[change_edition[:version]] = change_edition
+      end
+
+      def fetch_change_edition(version)
+        @change_editions[version]
+      end
+      alias_method :[], :fetch_change_edition
+
+      def save_to_file
+        File.write(filepath(@type), Psych.dump(to_h))
+      end
+
+      def to_h
+        {
+          "schema" => schema_name,
+          "change_edition" => change_editions.to_h,
+        }
+      end
+
+      private
+
+      def filepath(type)
+        File.join(
+          @stepmod_dir,
+          "data",
+          base_folder,
+          schema_name,
+          "#{MODULE_TYPES[type.to_sym] || schema_name}.changes.yaml",
+        )
+      end
+
+      def base_folder
+        if resource?
+          "resources"
+        else
+          "modules"
+        end
+      end
+    end
+  end
+end

data/lib/stepmod/utils/change_collection.rb

@@ -0,0 +1,50 @@
+require "stepmod/utils/change"
+
+module Stepmod
+  module Utils
+    class ChangeCollection
+      def initialize(stepmod_dir:)
+        @stepmod_dir = stepmod_dir
+        @changes = {}
+      end
+
+      def fetch_or_initialize(change, type)
+        schema = schema_name(change)
+
+        @changes[schema_identifier(schema, type)] ||= Change.new(
+          type: type,
+          stepmod_dir: @stepmod_dir,
+          schema_name: schema,
+        )
+      end
+
+      def fetch(change, type)
+        schema = schema_name(change)
+        @changes[schema_identifier(schema, type)]
+      end
+
+      def save_to_files
+        @changes.values.each(&:save_to_file)
+      end
+
+      def count
+        @changes.keys.count
+      end
+      alias_method :size, :count
+
+      def each(&block)
+        @changes.values.each(&block)
+      end
+
+      private
+
+      def schema_name(change)
+        change.is_a?(Stepmod::Utils::Change) ? change.schema_name : change
+      end
+
+      def schema_identifier(schema_name, type)
+        "#{schema_name}_#{type}"
+      end
+    end
+  end
+end

data/lib/stepmod/utils/change_edition.rb

@@ -0,0 +1,60 @@
+module Stepmod
+  module Utils
+    class ChangeEdition
+      attr_accessor :version, :description
+      attr_reader :additions, :modifications, :deletions, :mapping
+
+      def initialize(options)
+        @version = options[:version]
+        @description = options[:description]
+        self.additions = options[:additions] || []
+        self.modifications = options[:modifications] || []
+        self.deletions = options[:deletions] || []
+        self.mapping = options[:mapping] || []
+      end
+
+      def additions=(additions)
+        validate_type("additions", additions, Array)
+
+        @additions = additions
+      end
+
+      def modifications=(modifications)
+        validate_type("modifications", modifications, Array)
+
+        @modifications = modifications
+      end
+
+      def deletions=(deletions)
+        validate_type("deletions", deletions, Array)
+
+        @deletions = deletions
+      end
+
+      def mapping=(mapping)
+        validate_type("mapping", mapping, Array)
+
+        @mapping = mapping
+      end
+
+      def to_h
+        {
+          "version" => version,
+          "description" => description,
+          "additions" => additions,
+          "modifications" => modifications,
+          "deletions" => deletions,
+          "mapping" => mapping,
+        }.reject { |_k, v| v.nil? || v.empty? }
+      end
+
+      private
+
+      def validate_type(column, value, type)
+        error = "#{column} must be of type ::#{type}, Got ::#{value.class}"
+
+        raise error unless value.is_a?(type)
+      end
+    end
+  end
+end

data/lib/stepmod/utils/change_edition_collection.rb

@@ -0,0 +1,38 @@
+require "stepmod/utils/change_edition"
+
+module Stepmod
+  module Utils
+    class ChangeEditionCollection
+      def initialize
+        @collection = {}
+      end
+
+      def fetch_or_initialize(version)
+        @collection[version] ||=
+          Stepmod::Utils::ChangeEdition.new(version: version)
+      end
+
+      def to_h
+        @collection.values.map(&:to_h)
+      end
+
+      def []=(version, change_edition)
+        klass = Stepmod::Utils::ChangeEdition
+        @collection[version] = if change_edition.is_a?(klass)
+                                 change_edition
+                               else
+                                 klass.new(change_edition)
+                               end
+      end
+
+      def [](version)
+        @collection[version]
+      end
+
+      def count
+        @collection.values.count
+      end
+      alias_method :size, :count
+    end
+  end
+end