stepmod-utils 0.3.23 → 0.3.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96740d6528de476007a4f312fa8889887e77a27d4a693924531813188a3cd60b
-  data.tar.gz: 867240182ffe77c721fe7ffa8ee53b77383db9954cffa9c4de723e652ec90780
+  metadata.gz: 92d04b3c30620d813fbff40cfa84ed809f0129440dd1f6be45c4206aaa44a12e
+  data.tar.gz: 54289d1067ccba3a61cd0ca07bd5455081efe70aae946811cf8028f6e77afb9a
 SHA512:
-  metadata.gz: 6e5b71123d326d72e7f6f64269238e2c0db08cec4033794a1a104088d8ae3c0b7e9b005b2551bb2541b06d676fb6959120a7bd1aac30c3cc63068e581206d8c3
-  data.tar.gz: 9aba582cd94d7c7b85fd9bc8ad2e718c0bf23f6609a86de46393c55a85bda496fecbe9f8c847fd073a5aa1ca351148917520a490d695769fde21d118428fbe98
+  metadata.gz: e83145314175c1797f8de0fe08bdbf3ba95e06346b49f80beb3cdadb31d1b2423cb60230d869b91850de3685b0fb6c805a2b4ddebbfd72ff2b55ffde234f4d9f
+  data.tar.gz: 309a341bead6e54225f14101c3bea989f2f7873409c4d8e1fedbc6e8b2144d3e21ed2034c5ebcfd76279d73234a84e8d0d67c86fb77e1e1b715f6f1ba2ad661b

data/.rubocop.yml

@@ -6,5 +6,8 @@ inherit_from:
 # local repo-specific modifications
 # ...
 
+Style/MultilineMethodCallIndentation:
+  EnforcedStyle: aligned
+
 AllCops:
   TargetRubyVersion: 2.5

data/exe/stepmod-extract-yaml-terms

@@ -0,0 +1,149 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require "pry"
+# resolve bin path, ignoring symlinks
+require "pathname"
+bin_file = Pathname.new(__FILE__).realpath
+
+# add self to libpath
+$:.unshift File.expand_path("../../lib", bin_file)
+
+# Fixes https://github.com/rubygems/rubygems/issues/1420
+require "rubygems/specification"
+
+module Gem
+  class Specification
+    def this; self; end
+  end
+end
+
+require "bundler/setup"
+require_relative "../lib/stepmod/utils/terms_extractor"
+require "optparse"
+
+def log(message)
+  puts "[stepmod-utils] #{message}"
+end
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.on(
+    "-p",
+    "--path STEPMOD_DATA_PATH",
+    String,
+    "Path to STEPmod data directory",
+  ) do |path|
+    options[:stepmod_dir] = path
+  end
+
+  opts.on(
+    "-i",
+    "--index INDEX_PATH",
+    String,
+    "Path to repository_index.xml",
+  ) do |path|
+    unless path.nil?
+      options[:index_path] = Pathname.new(path).to_s
+    end
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+stepmod_dir = options[:stepmod_dir]
+if stepmod_dir.nil?
+  raise StandardError.new(
+    "STEPmod data path not set, set with the `-p` option.",
+  )
+else
+  log "STEPmod data path: `#{stepmod_dir}`"
+end
+
+default_index_path = File.join(stepmod_dir, "repository_index.xml")
+index_path = options[:index_path] || default_index_path
+if File.exists?(index_path)
+  log "Repository index path: `#{index_path}`"
+else
+  raise StandardError.new(
+    "Index file not present at #{index_path}, set with the `-i` option.",
+  )
+end
+
+_general_concepts,
+resource_concepts,
+_parsed_bibliography,
+_part_concepts,
+part_resources,
+part_modules = Stepmod::Utils::TermsExtractor.call(stepmod_dir, index_path)
+
+def part_to_title(bibdata)
+  {
+    41 => "Fundamentals of product description and support",
+    42 => "Geometric and topological representation",
+    43 => "Foundation representation",
+    44 => "Product structure, concept and configuration",
+    45 => "Material and other engineering properties",
+    46 => "Visual presentation",
+    47 => "Shape tolerance",
+    51 => "Mathematical representation",
+  }[bibdata.part.to_i] || bibdata.title_en
+end
+
+# rubocop:disable Layout/LineLength
+IMAGE_REPLACEMENTS = {
+  "image::eq01.gif[]" => "stem:[H(A,B) = max {h(A, B), h(B,A)}]",
+  "image::eq02.gif[]" => "stem:[max_{a in A} { min_{b in B} d(a,b) }]",
+  "image::vector_z_c.gif[]" => "stem:[bar z_{c}]",
+  "image::one_direction_repeat_factor_expression.gif[]" => "stem:[I + k cdot R; k = -1, 1]",
+  "image::two_direction_repeat_factor_expression.gif[]" => "stem:[I + k_1 cdot R_1 + k_2 cdot R_2; k_1, k_2 = -1, 0, 1, k^2_1 + k^2_2 != 0]",
+}.freeze
+
+TEXT_REPLACEMENTS = {
+  ' (see <module_ref linkend="ply_orientation_specification:4_entities:figure:f2"> Figure 2</module_ref>)' => "",
+  ' (see <module_ref linkend="ply_orientation_specification:4_entities:figure:f3"> Figure 3</module_ref>)' => "",
+}.freeze
+# rubocop:enable Layout/LineLength
+
+def replace_content(content)
+  IMAGE_REPLACEMENTS.each_pair do |k, v|
+    content.gsub!(k, v)
+  end
+  TEXT_REPLACEMENTS.each_pair do |k, v|
+    content.gsub!(k, v)
+  end
+
+  content
+end
+
+part_resources.each do |(_bibdata, current_part_resources)|
+  current_part_resources.save_to_files("./")
+end
+log "INFO: part_resources written to yaml files"
+
+
+log "INFO: written summary file to: 04x-stepmod-entities-resources.adoc"
+part_modules.sort_by do |(bibdata, _part_modules_arm, _part_modules_mim)|
+  bibdata.part.to_i
+end.each do |(_bibdata, part_modules_arm, part_modules_mim)|
+  unless part_modules_arm.empty?
+    part_modules_arm.values.map do |managed_concept|
+      managed_concept.save_to_files("./")
+    end
+  end
+
+  unless part_modules_mim.empty?
+    part_modules_mim.values.map do |managed_concept|
+      managed_concept.save_to_files("./")
+    end
+  end
+end
+log "INFO: part_modules written to yaml files"
+
+resource_concepts.save_to_files("./")
+log "INFO: resource_concepts written to yaml files"

data/lib/stepmod/utils/concept.rb

@@ -8,6 +8,10 @@ module Stepmod
         reference_anchor
         converted_definition
         file_path
+        schema
+        part
+        domain
+        document
       )
 
       # TODO: converted_definition is not supposed to be an attribute, it is
@@ -40,8 +44,8 @@ module Stepmod
           # TODO: `designations:` should include the `alt:[...]` terms here,
           # they are now only included in definition_xml_converted_definition.
           new(
-            designations: [designation],
-            definition: definition,
+            designations: designation,
+            definition: [definition],
             converted_definition: converted_definition,
             id: "#{reference_anchor}.#{reference_clause}",
             reference_anchor: reference_anchor,
@@ -65,11 +69,16 @@ module Stepmod
               definition_xml.xpath(".//term").first
             )
 
-          {
-            # [4..-1] because we want to skip the initial `=== {title}`
-            accepted: term[4..-1],
-            alt: alts,
-          }
+          # [4..-1] because we want to skip the initial `=== {title}`
+          designations = [
+            { "designation" => term[4..-1], "type" => "expression", "normative_status" => "preferred" },
+          ]
+
+          alts.each do |alt|
+            designations << { "designation" => alt, "type" => "expression" }
+          end
+
+          designations
         end
 
         def definition_xml_definition(definition_xml, reference_anchor)
@@ -95,12 +104,24 @@ module Stepmod
         end
 
         def definition_xml_converted_definition(designation, definition)
-          if designation[:alt].length.positive?
-            alt_notation = "alt:[#{designation[:alt].map(&:strip).join(',')}]"
+          accepted_designation = designation.select do |des|
+            des["normative_status"] == "preferred"
+          end
+
+          alt_designations = designation.reject do |des|
+            des["normative_status"] == "preferred"
+          end
+
+          if alt_designations.length.positive?
+            alt_designations_text = alt_designations.map do |d|
+              d["designation"].strip
+            end.join(",")
+
+            alt_notation = "alt:[#{alt_designations_text}]"
           end
 
           result = <<~TEXT
-            === #{designation[:accepted].strip}
+            === #{accepted_designation.map { |d| d['designation'].strip }.join(',')}
           TEXT
 
           if alt_notation
@@ -117,6 +138,15 @@ module Stepmod
         end
       end
 
+      def to_h
+        super.merge({
+          "domain" => domain,
+          "part" => part,
+          "schema" => schema,
+          "document" => document,
+        }.compact)
+      end
+
       def to_mn_adoc
         <<~TEXT
           // STEPmod path:#{file_path.empty? ? '' : " #{file_path}"}

data/lib/stepmod/utils/express_bibdata.rb

@@ -0,0 +1,111 @@
+module Stepmod
+  module Utils
+    class ExpressBibdata
+      DOCNUMBER = 10303
+
+      attr_accessor *%w(
+        type doctype part title_en version pub_year pubid published_info number
+      )
+
+      def initialize(schema:)
+        # module, resource, application_protocol, business_object_model
+        # @type = document.name
+
+        # raise "UnknownFileError" unless %w(module resource
+        #                                    application_protocol business_object_model).include?(@type)
+
+        @published_info = schema.find("__published_in")&.remarks&.first
+        @number = schema.find("__identifier")&.remarks&.first&.split("N")&.last
+        @schema = schema
+
+        if !published_info.nil?
+          @pubid = Pubid::Iso::Identifier.parse(published_info)
+
+          @part = pubid.part
+          @version = pubid.edition
+          @pub_year = pubid.year
+        elsif !schema.version.nil?
+          @part = schema.version.items.find { |i| i.name == "part" }.value
+          @version = schema.version.items.find { |i| i.name == "part" }.value
+          @pub_year = schema.version.items.find { |i| i.name == "part" }.value
+        else
+          raise "PublishedInfoNotFound"
+        end
+
+        @doctype = schema.find("__status")&.remarks&.first
+        self
+      end
+
+      def title_en
+        @title_en ||= @schema.find("__title")
+                             .remarks
+                             .first
+                             .gsub("_", " ")
+                             .capitalize
+                             .gsub("2d", "2D")
+                             .gsub("3d", "3D")
+      end
+
+      def docid
+        no_date = case doctype
+                  when "IS"
+                    "ISO #{DOCNUMBER}-#{part}"
+                  when "WD"
+                    "ISO/WD #{DOCNUMBER}-#{part}"
+                  when "CD"
+                    "ISO/CD #{DOCNUMBER}-#{part}"
+                  when "DIS"
+                    "ISO/DIS #{DOCNUMBER}-#{part}"
+                  when "FDIS"
+                    "ISO/FDIS #{DOCNUMBER}-#{part}"
+                  when "TS"
+                    "ISO/TS #{DOCNUMBER}-#{part}"
+                  when "CD-TS"
+                    "ISO/CD TS #{DOCNUMBER}-#{part}"
+                  else
+                    "UNKNOWN DOCTYPE: (#{doctype})"
+                  end
+
+        if pub_year
+          "#{no_date}:#{pub_year}"
+        else
+          no_date
+        end
+      end
+
+      def part_title
+        case part
+        when [200..299]
+          "Application protocol: #{title_en}"
+        when [300..399]
+          "Abstract test suite: #{title_en}"
+        when [400..499]
+          "Application module: #{title_en}"
+        when [500..599]
+          "Application interpreted construct: #{title_en}"
+        when [1000..1799]
+          "Application module: #{title_en}"
+        else
+          title_en
+        end
+      end
+
+      def full_title
+        "Industrial automation systems and integration -- Product data" \
+          " representation and exchange -- Part #{part}: #{part_title}"
+      end
+
+      def anchor
+        docid.gsub("/", "-").gsub(" ", "_").gsub(":", "_")
+      end
+
+      def to_mn_adoc
+        if title_en
+          "* [[[#{anchor},#{docid}]]], _#{full_title}_"
+        else
+          "* [[[#{anchor},#{docid}]]]"
+        end
+      end
+    end
+  end
+end

data/lib/stepmod/utils/stepmod_file_annotator.rb

@@ -5,6 +5,9 @@ require "stepmod/utils/converters/express_note"
 require "stepmod/utils/converters/express_example"
 require "stepmod/utils/converters/express_figure"
 require "stepmod/utils/converters/express_table"
+require "expressir"
+require "expressir/express/parser"
+require "pubid-iso"
 
 module Stepmod
   module Utils
@@ -17,26 +20,37 @@ module Stepmod
         @express_file = express_file
         @resource_docs_cache_file = resource_docs_cache_file
         @stepmod_dir = stepmod_dir || Dir.pwd
+        @added_bibdata = {}
+
+        @schema_name = Expressir::Express::Parser.from_file(express_file)
+                                                 .schemas
+                                                 .first
+                                                 .id
       end
 
       def call
         match = File.basename(express_file).match('^(arm|mim|bom)\.exp$')
         descriptions_base = match ? "#{match.captures[0]}_descriptions.xml" : "descriptions.xml"
+
         descriptions_file = File.join(File.dirname(express_file),
                                       descriptions_base)
+
         output_express = File.read(express_file)
-        resource_docs_cache = JSON.parse(File.read(resource_docs_cache_file))
+        converted_description = ""
+        base_linked = ""
 
         if File.exists?(descriptions_file)
           descriptions = Nokogiri::XML(File.read(descriptions_file)).root
           added_resource_descriptions = {}
+
           descriptions.xpath("ext_description").each do |description|
             # Add base resource from linked path if exists, eg "language_schema.language.wr:WR1" -> "language_schema"
             base_linked = description["linkend"].to_s.split(".").first
+
             if added_resource_descriptions[base_linked].nil?
               base_reource_doc_dir = resource_docs_cache[description["linkend"].to_s.split(".").first]
               if base_reource_doc_dir
-                output_express << convert_from_resource_file(
+                converted_description << convert_from_resource_file(
                   base_reource_doc_dir, stepmod_dir, base_linked, descriptions_file
                 )
               end
@@ -47,7 +61,7 @@ module Stepmod
             # when a schema description is available from resource.xml and also descriptions.xml, the description from resource.xml is only used.
             # https://github.com/metanorma/annotated-express/issues/32#issuecomment-792609078
             if description.text.strip.length.positive? && resource_docs_dir.nil?
-              output_express << convert_from_description_text(
+              converted_description << convert_from_description_text(
                 descriptions_file, description
               )
             end
@@ -60,7 +74,28 @@ module Stepmod
           end
         end
 
-        output_express
+        bib_file_name = extract_bib_file_name(match, resource_docs_cache[@schema_name || ""])
+        bib_file = if match
+                     File.join(File.dirname(express_file), bib_file_name)
+                   else
+                     resource_docs_file_path(stepmod_dir, bib_file_name)
+                   end
+
+        output_express << if bib_file && File.exists?(bib_file)
+                            prepend_bibdata(
+                              converted_description || "",
+                              # bib_file will not be present for resouces
+                              # that are not in resource_docs cache.
+                              # e.g hierarchy_schema
+                              bib_file,
+                              @schema_name,
+                              match,
+                            )
+                          else
+                            converted_description
+                          end
+
+        sanitize(output_express)
       rescue StandardError => e
         puts "[ERROR]!!! #{e.message}"
         puts e.backtrace
@@ -68,6 +103,14 @@ module Stepmod
 
       private
 
+      def sanitize(file_content)
+        file_content.gsub("(*)", "(`*`)")
+      end
+
+      def resource_docs_cache
+        @resource_docs_cache ||= JSON.parse(File.read(resource_docs_cache_file))
+      end
+
       def convert_from_description_text(descriptions_file, description)
         Dir.chdir(File.dirname(descriptions_file)) do
           wrapper = "<ext_descriptions>#{description}</ext_descriptions>"
@@ -119,10 +162,146 @@ module Stepmod
         end
       end
 
+      def prepend_bibdata(description, bibdata_file, schema_and_entity, match)
+        bib = Nokogiri::XML(File.read(bibdata_file)).root
+        bibdata = extract_bib_data(match, bib, schema_and_entity)
+
+        return description.to_s if @added_bibdata[schema_and_entity]
+
+        published_in = <<~PUBLISHED_IN
+
+          (*"#{schema_and_entity}.__published_in"
+          #{bibdata[:identifier]}
+          *)
+        PUBLISHED_IN
+
+        identifier = <<~IDENTIFIER if bibdata[:number]
+          (*"#{schema_and_entity}.__identifier"
+          ISO/TC 184/SC 4/WG 12 N#{bibdata[:number]}
+          *)
+        IDENTIFIER
+
+        supersedes = <<~SUPERSEDES if bibdata[:supersedes_concept]
+          (*"#{schema_and_entity}.__supersedes"
+          ISO/TC 184/SC 4/WG 12 N#{bibdata[:supersedes_concept]}
+          *)
+        SUPERSEDES
+
+        status = <<~STATUS if bibdata[:status]
+          (*"#{schema_and_entity}.__status"
+          #{bibdata[:status]}
+          *)
+        STATUS
+
+        title = <<~TITLE if bibdata[:title]
+          (*"#{schema_and_entity}.__title"
+          #{bibdata[:title]}
+          *)
+        TITLE
+
+        document = <<~DOCUMENT if bibdata_file
+          (*"#{schema_and_entity}.__schema_file"
+          #{Pathname(bibdata_file).relative_path_from(@stepmod_dir)}
+          *)
+        DOCUMENT
+
+        @added_bibdata[schema_and_entity] = true
+
+        [
+          published_in,
+          identifier,
+          supersedes,
+          status,
+          title,
+          description,
+          document,
+        ].compact.join("\n")
+      end
+
+      def module?(match)
+        match && %w[arm mim].include?(match.captures[0])
+      end
+
+      def bom?(match)
+        match && %w[bom].include?(match.captures[0])
+      end
+
+      def extract_bib_file_name(match, default_file_name = "")
+        return default_file_name || "" unless match
+
+        if %w[arm mim].include?(match.captures[0])
+          "module.xml"
+        else
+          "business_object_model.xml"
+        end
+      end
+
+      def extract_bib_data(match, bib, schema_and_entity)
+        return resource_bib_data(bib, schema_and_entity) unless match
+
+        if module?(match)
+          module_bib_data(bib, match.captures[0])
+        elsif bom?(match)
+          bom_bib_data(bib)
+        end
+      end
+
+      def identifier(bib)
+        part = bib.attributes["part"].value
+        year = bib.attributes["publication.year"].value
+
+        # year="tbd" in data/modules/geometric_tolerance/module.xml and
+        # probabaly in some other places as well
+        year = "" if year == "tbd"
+        edition = bib.attributes["version"].value
+
+        pubid = Pubid::Iso::Identifier.new(
+          publisher: "ISO",
+          number: 10303,
+        )
+
+        pubid.part = part if part && !part.empty?
+        pubid.year = year.split("-").first if year && !year.empty?
+        pubid.edition = edition if edition && !edition.empty?
+
+        pubid.to_s(with_edition: true)
+      end
+
+      def resource_bib_data(bib, schema_and_entity)
+        schema = bib.xpath("schema[@name='#{schema_and_entity}']").first
+
+        {
+          identifier: identifier(bib),
+          number: schema.attributes["number"],
+          supersedes_concept: schema.attributes["number.supersedes"],
+          status: bib.attributes["status"],
+          title: bib.attributes["title"] || bib.attributes["name"],
+        }
+      end
+
+      def module_bib_data(bib, type)
+        {
+          identifier: identifier(bib),
+          number: bib.attributes["wg.number.#{type}"],
+          supersedes_concept: bib.attributes["wg.number.#{type}.supersedes"],
+          status: bib.attributes["status"],
+          title: bib.attributes["title"] || bib.attributes["name"],
+        }
+      end
+
+      def bom_bib_data(bib)
+        {
+          identifier: identifier(bib),
+          number: bib.attributes["wg.number.bom.exp"],
+          supersedes_concept: bib.attributes["wg.number.bom.supersedes"],
+          status: bib.attributes["status"],
+          title: bib.attributes["title"] || bib.attributes["name"],
+        }
+      end
+
       def convert_from_resource_file(resource_docs_dir, stepmod_dir, linked, descriptions_file)
-        resource_docs_file = File.join(stepmod_dir, "data/resource_docs",
-                                       resource_docs_dir, "resource.xml")
-        # puts(resource_docs_file)
+        resource_docs_file = resource_docs_file_path(stepmod_dir, resource_docs_dir)
+
         resource_docs = Nokogiri::XML(File.read(resource_docs_file)).root
         schema = resource_docs.xpath("schema[@name='#{linked}']")
 
@@ -133,11 +312,20 @@ module Stepmod
             wrapper,
             {
               no_notes_examples: false,
-              schema_and_entity: linked
-            }
+              schema_and_entity: linked,
+            },
           )
         end
       end
+
+      def resource_docs_file_path(stepmod_dir, resource_docs_dir)
+        File.join(
+          stepmod_dir,
+          "data/resource_docs",
+          resource_docs_dir,
+          "resource.xml",
+        )
+      end
     end
   end
 end

data/lib/stepmod/utils/term.rb

@@ -7,9 +7,9 @@ module Stepmod
       attr_accessor :acronym
 
       def to_mn_adoc
-        mn_adoc = ["=== #{definition}"]
+        mn_adoc = ["=== #{definition.map(&:content).join}"]
         mn_adoc << "\nalt:[#{acronym}]" if acronym
-        mn_adoc << "\n\n#{designations.join(", ")}" if designations&.any?
+        mn_adoc << "\n\n#{designations.map(&:designation).join(", ")}" if designations&.any?
 
         mn_adoc.join
       end
@@ -18,15 +18,28 @@ module Stepmod
         def from_h(hash)
           _, definition, acronym = treat_acronym(hash["definition"])
 
-          hash["definition"] = definition
+          hash["definition"] = [definition]
+
           hash["acronym"] = acronym.gsub(/\(|\)/, "") if acronym
-          hash["designations"] = hash["synonyms"]
+          add_designations(hash, hash["synonyms"]) if hash["synonyms"]
 
-          super(hash.reject { |k, _| k == "synonyms" })
+          new(hash.reject { |k, _| k == "synonyms" })
         end
 
         private
 
+        def add_designations(hash, synonyms)
+          hash["designations"] ||= []
+          hash["designations"] << designation_hash(synonyms) if synonyms
+        end
+
+        def designation_hash(value, type = "expression")
+          {
+            "designation" => value,
+            "type" => type,
+          }
+        end
+
         def treat_acronym(term_def)
           return [nil, term_def.strip, nil] unless term_def.match?(/.+\(.+?\)$/)