expressir 2.3.6 → 2.4.0

data/lib/expressir/express/remark_scanner.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Scans EXPRESS source text and extracts all remarks (tail and embedded),
+    # correctly tracking embedded-remark nesting so that inline `--` inside
+    # a `(* ... *)` documentation block is NOT mistaken for a tail remark.
+    #
+    # Single responsibility: source string -> Array<Remark>. The scanner has
+    # no knowledge of the model; attaching remarks to model elements is the
+    # job of {RemarkAttacher}.
+    #
+    # Reference: ISO 10303-11 §7.1.6.
+    #   tail_remark     = '--' [ remark_tag ] { <any char except newline> } '\n'
+    #   embedded_remark = '(*' [ remark_tag ]
+    #                       { <any char> | nested embedded_remark } '*)'
+    #
+    # The scanner is a single-pass state machine with three states:
+    #   :top          — outside any remark; look for `--` or `(*`.
+    #   :in_tail       — inside a tail remark; look only for the next newline.
+    #   :in_embedded   — inside one or more nested embedded remarks; look
+    #                    for the matching `*)` or a nested `(*`, ignoring `--`.
+    class RemarkScanner
+      # Immutable value object describing a single extracted remark.
+      Remark = Struct.new(:position, :line, :text, :tag, :format, keyword_init: true) do
+        def tail?
+          format == TAIL_FORMAT
+        end
+
+        def embedded?
+          format == EMBEDDED_FORMAT
+        end
+
+        def tagged?
+          !tag.nil? && !tag.empty?
+        end
+      end
+
+      # Format discriminator values.
+      TAIL_FORMAT = "tail"
+      EMBEDDED_FORMAT = "embedded"
+
+      # Lexical markers recognised by EXPRESS.
+      TAIL_MARKER = "--"
+      EMBEDDED_OPEN = "(*"
+      EMBEDDED_CLOSE = "*)"
+
+      # Tag forms (see ISO 10303-11 §7.1.6.3 Remark tag).
+      # IP-style informal-proposition tags are recognised only in tail remarks
+      # (matching historic behaviour); embedded remarks use the quoted form.
+      INFORMAL_PROPOSITION_TAG = /\A(IP\d+):\s*(.*)\z/m
+      QUOTED_TAG = /\A"([^"]*)"\s*(.*)\z/m
+
+      # @param source [String] EXPRESS source text (any encoding).
+      def initialize(source)
+        @source = source
+        @source_bytes = source.b
+        @line_map = LineMap.new(@source_bytes)
+      end
+
+      # Returns an Array of {Remark}, in source order (by byte position).
+      # @return [Array<Remark>]
+      def scan
+        run_state_machine
+      end
+
+      alias call scan
+
+      private
+
+      def run_state_machine
+        remarks = []
+        src = @source_bytes
+        pos = 0
+
+        state = :top
+        embedded_depth = 0
+        tail_start = nil      # byte position of first content char after "--"
+        tail_line = nil
+        embedded_start = nil  # byte position of first content char after "(*"
+        embedded_line = nil
+
+        while pos < src.bytesize
+          case state
+          when :in_tail
+            nl = src.index("\n", pos)
+            if nl
+              emit_tail(remarks, tail_start, nl, tail_line)
+              pos = nl + 1
+              state = :top
+            else
+              emit_tail(remarks, tail_start, src.bytesize, tail_line)
+              return remarks
+            end
+          when :in_embedded
+            closing = src.index(EMBEDDED_CLOSE, pos)
+            nesting = src.index(EMBEDDED_OPEN, pos)
+            if nesting && (!closing || nesting < closing)
+              embedded_depth += 1
+              pos = nesting + EMBEDDED_OPEN.bytesize
+            elsif closing
+              embedded_depth -= 1
+              pos = closing + EMBEDDED_CLOSE.bytesize
+              if embedded_depth.zero?
+                emit_embedded(remarks, embedded_start, closing, embedded_line)
+                state = :top
+              end
+            else
+              # Unclosed embedded remark — terminate scan gracefully.
+              return remarks
+            end
+          else # :top
+            next_embed = src.index(EMBEDDED_OPEN, pos)
+            next_tail = src.index(TAIL_MARKER, pos)
+            if next_embed.nil? && next_tail.nil?
+              return remarks
+            elsif next_embed && (next_tail.nil? || next_embed <= next_tail)
+              embedded_start = next_embed + EMBEDDED_OPEN.bytesize
+              embedded_line = @line_map.line_number(next_embed)
+              embedded_depth = 1
+              state = :in_embedded
+              pos = next_embed + EMBEDDED_OPEN.bytesize
+            else
+              tail_start = next_tail + TAIL_MARKER.bytesize
+              tail_line = @line_map.line_number(next_tail)
+              state = :in_tail
+              pos = next_tail + TAIL_MARKER.bytesize
+            end
+          end
+        end
+
+        remarks
+      end
+
+      def emit_tail(remarks, content_start, content_end, line)
+        # Slice content from the original (UTF-8) source so the returned text
+        # preserves the source encoding. Byte offsets come from the BINARY copy.
+        raw = @source.byteslice(content_start...content_end)
+        tag, content = parse_tail(raw.strip)
+        remarks << Remark.new(
+          position: content_start - TAIL_MARKER.bytesize,
+          line: line,
+          text: content,
+          tag: tag,
+          format: TAIL_FORMAT,
+        )
+      end
+
+      def emit_embedded(remarks, content_start, content_end, line)
+        raw = @source.byteslice(content_start...content_end)
+        tag, content = parse_embedded(raw)
+        remarks << Remark.new(
+          position: content_start - EMBEDDED_OPEN.bytesize,
+          line: line,
+          text: content,
+          tag: tag,
+          format: EMBEDDED_FORMAT,
+        )
+      end
+
+      # Tail remarks support both quoted tags ("schema.entity" ...) and the
+      # shorthand IP-tag form (IP1: ...). Returns [tag, content]; for untagged
+      # remarks tag is nil and content is the stripped text.
+      def parse_tail(text)
+        match = text.match(INFORMAL_PROPOSITION_TAG) || text.match(QUOTED_TAG)
+        match ? match.captures : [nil, text]
+      end
+
+      # Embedded remarks only use the quoted tag form.
+      def parse_embedded(content)
+        stripped = content.strip
+        if (m = stripped.match(QUOTED_TAG))
+          [m[1], m[2]]
+        else
+          [nil, stripped]
+        end
+      end
+    end
+  end
+end

data/lib/expressir/express/schema_head_formatter.rb

@@ -8,14 +8,12 @@ module Expressir
     module SchemaHeadFormatter
       # @!visibility private
       def self.included(mod)
-        if !mod.superclass.private_method_defined?(:format_declarations_schema) || !mod.superclass.private_method_defined?(:format_declarations_schema_head)
+        unless mod.superclass <= Expressir::Express::Formatter
           raise Error::FormatterMethodMissingError.new("SchemaHeadFormatter",
                                                        "format_declarations_schema/format_declarations_schema_head")
         end
       end
 
-      private
-
       def format_declarations_schema(node)
         format_declarations_schema_head(node)
       end

data/lib/expressir/express.rb

@@ -12,10 +12,12 @@ module Expressir
     autoload :Formatter, "#{__dir__}/express/formatter"
     autoload :Formatters, "#{__dir__}/express/formatters"
     autoload :HyperlinkFormatter, "#{__dir__}/express/hyperlink_formatter"
+    autoload :LineMap, "#{__dir__}/express/line_map"
     autoload :ModelVisitor, "#{__dir__}/express/model_visitor"
     autoload :Parser, "#{__dir__}/express/parser"
     autoload :PrettyFormatter, "#{__dir__}/express/pretty_formatter"
     autoload :RemarkAttacher, "#{__dir__}/express/remark_attacher"
+    autoload :RemarkScanner, "#{__dir__}/express/remark_scanner"
     autoload :ResolveReferencesModelVisitor,
              "#{__dir__}/express/resolve_references_model_visitor"
     autoload :SchemaHeadFormatter, "#{__dir__}/express/schema_head_formatter"

data/lib/expressir/model/concerns.rb

@@ -2,10 +2,16 @@
 
 module Expressir
   module Model
+    # Marker for types that have an id attribute
+    module HasId; end
+
     # Marker for types that can have remark_items children
     # These are types where RemarkItem children can be created
     module HasRemarkItems; end
 
+    # Marker for types that have a `remarks` string collection
+    module HasRemarks; end
+
     # Marker for scope containers (can contain declarations)
     # Includes schemas, functions, procedures, rules, entities, types, and files
     module ScopeContainer; end

data/lib/expressir/model/declarations/interface_item.rb

@@ -4,6 +4,8 @@ module Expressir
       # Specified in ISO 10303-11:2004
       # - section 11 Interface specification
       class InterfaceItem < ModelElement
+        include HasId
+
         attribute :ref, ModelElement
         attribute :id, :string
         attribute :_class, :string, default: -> { self.class.name }

data/lib/expressir/model/declarations/interfaced_item.rb

@@ -4,6 +4,9 @@ module Expressir
       # Specified in ISO 10303-11:2004
       # - section 11 Interface specification
       class InterfacedItem < ModelElement
+        include HasId
+        include HasRemarks
+
         attribute :id, :string
         attribute :remarks, :string, collection: true
         attribute :remark_items, RemarkItem, collection: true

data/lib/expressir/model/declarations/remark_item.rb

@@ -3,6 +3,9 @@ module Expressir
     module Declarations
       # Implicit item with remarks
       class RemarkItem < ModelElement
+        include HasId
+        include HasRemarks
+
         attribute :id, :string
         attribute :remarks, :string, collection: true
         attribute :_class, :string, default: -> { self.class.name }

data/lib/expressir/model/declarations/schema.rb

@@ -70,7 +70,7 @@ module Expressir
         end
 
         def full_source
-          Expressir::Express::Formatter.format(self)
+          @full_source ||= Expressir::Express::Formatter.format(self)
         end
 
         def formatted
@@ -78,11 +78,13 @@ module Expressir
         end
 
         def source
-          formatter = Class.new(Expressir::Express::Formatter) do
-            include Expressir::Express::SchemaHeadFormatter
-            include Expressir::Express::HyperlinkFormatter
+          @source ||= begin
+            formatter = Class.new(Expressir::Express::Formatter) do
+              include Expressir::Express::SchemaHeadFormatter
+              include Expressir::Express::HyperlinkFormatter
+            end
+            formatter.format(self)
           end
-          formatter.format(self)
         end
 
         private

data/lib/expressir/model/exp_file.rb

@@ -11,6 +11,7 @@ module Expressir
     # A Repository contains multiple ExpFile instances when parsing multiple files,
     # or a single ExpFile when parsing a single file.
     class ExpFile < ModelElement
+      include HasId
       include ScopeContainer
 
       attribute :path, :string

data/lib/expressir/model/identifier.rb

@@ -2,8 +2,10 @@ module Expressir
   module Model
     module Identifier
       def self.included(mod)
-        # Auto-include marker - all Identifier types can have remark_items
+        # Auto-include markers - all Identifier types have id and can have remark_items and remarks
+        mod.include(HasId)
         mod.include(HasRemarkItems)
+        mod.include(HasRemarks)
 
         mod.attribute :id, :string
         mod.attribute :remarks, :string, collection: true

data/lib/expressir/model/model_element.rb

@@ -120,7 +120,7 @@ module Expressir
         loop do
           # Skip adding the id if this is a RemarkItem that belongs to an InformalPropositionRule
           # and has the same id as its parent
-          if !(current_node.is_a? References::SimpleReference) && current_node.class.method_defined?(:id) &&
+          if !(current_node.is_a? References::SimpleReference) && current_node.is_a?(HasId) &&
               !(is_a?(Declarations::RemarkItem) &&
                 parent.is_a?(Declarations::InformalPropositionRule) &&
                 id == parent.id)
@@ -200,7 +200,7 @@ module Expressir
         self.untagged_remarks ||= []
         return unless remark_info.is_a?(RemarkInfo)
 
-        self.untagged_remarks << remark_info
+        untagged_remarks << remark_info
       end
 
       private
@@ -208,7 +208,7 @@ module Expressir
       # @return [nil]
       def attach_parent_to_children
         self.class.attributes.each_pair do |symbol, _lutaml_attr|
-          value = send(symbol)
+          value = public_send(symbol)
 
           case value
           when Array

data/lib/expressir/model/repository.rb

@@ -24,6 +24,14 @@ module Expressir
       # Index instances (lazy-loaded)
       attr_reader :entity_index, :type_index, :reference_index
 
+      # Restore deserialized indexes (used by Package::Reader)
+      def restore_indexes(entity_index: nil, type_index: nil,
+reference_index: nil)
+        @entity_index = entity_index if entity_index
+        @type_index = type_index if type_index
+        @reference_index = reference_index if reference_index
+      end
+
       # Internal schema storage for direct manipulation
       attr_reader :_schemas
 

data/lib/expressir/model/search_engine.rb

@@ -327,7 +327,7 @@ module Expressir
         # Guard against nil schemas array
         return [] unless schemas
 
-        schemas.flat_map { |s| s.send(method) || [] }
+        schemas.flat_map { |s| s.public_send(method) || [] }
       end
 
       # Collect all attributes from entities

data/lib/expressir/package/reader.rb

@@ -132,32 +132,17 @@ module Expressir
       # @param repository [Model::Repository] Repository instance
       # @return [void]
       def load_indexes(zip, repository)
-        # Load entity index
-        entity_index_entry = zip.find_entry("entity_index.marshal")
-        if entity_index_entry
-          repository.instance_variable_set(
-            :@entity_index,
-            Marshal.load(entity_index_entry.get_input_stream.read),
-          )
+        indexes = {}
+
+        %w[entity_index type_index reference_index].each do |name|
+          entry = zip.find_entry("#{name}.marshal")
+          if entry
+            indexes[name.to_sym] =
+              Marshal.load(entry.get_input_stream.read)
+          end
         end
 
-        # Load type index
-        type_index_entry = zip.find_entry("type_index.marshal")
-        if type_index_entry
-          repository.instance_variable_set(
-            :@type_index,
-            Marshal.load(type_index_entry.get_input_stream.read),
-          )
-        end
-
-        # Load reference index
-        reference_index_entry = zip.find_entry("reference_index.marshal")
-        if reference_index_entry
-          repository.instance_variable_set(
-            :@reference_index,
-            Marshal.load(reference_index_entry.get_input_stream.read),
-          )
-        end
+        repository.restore_indexes(**indexes) unless indexes.empty?
       end
     end
   end

data/lib/expressir/version.rb

@@ -1,3 +1,3 @@
 module Expressir
-  VERSION = "2.3.6".freeze
+  VERSION = "2.4.0".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: expressir
 version: !ruby/object:Gem::Version
-  version: 2.3.6
+  version: 2.4.0
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-14 00:00:00.000000000 Z
+date: 2026-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -233,6 +233,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
+- CHANGELOG.md
 - Gemfile
 - README.adoc
 - Rakefile
@@ -263,6 +264,7 @@ files:
 - docs/_guides/ler/loading-packages.adoc
 - docs/_guides/ler/package-formats.adoc
 - docs/_guides/ler/querying-packages.adoc
+- docs/_guides/ler/step-packages.adoc
 - docs/_guides/ler/validating-packages.adoc
 - docs/_guides/liquid/basic-templates.adoc
 - docs/_guides/liquid/documentation-generation.adoc
@@ -396,10 +398,12 @@ files:
 - lib/expressir/express/formatters/statements_formatter.rb
 - lib/expressir/express/formatters/supertype_expressions_formatter.rb
 - lib/expressir/express/hyperlink_formatter.rb
+- lib/expressir/express/line_map.rb
 - lib/expressir/express/model_visitor.rb
 - lib/expressir/express/parser.rb
 - lib/expressir/express/pretty_formatter.rb
 - lib/expressir/express/remark_attacher.rb
+- lib/expressir/express/remark_scanner.rb
 - lib/expressir/express/resolve_references_model_visitor.rb
 - lib/expressir/express/schema_head_formatter.rb
 - lib/expressir/express/streaming_builder.rb