expressir 2.4.3 → 2.4.4

data/lib/expressir/express/checker.rb

@@ -0,0 +1,458 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Semantic checks modeled on eeng kernel/check.lisp + check-notes.lisp
+    # (expressir parity-ee stage 07). Each note preserves the eeng note id
+    # so oracle differentials stay cross-referenceable.
+    #
+    # Coverage is a deliberate subset of eeng's check-p11 catalog:
+    #   check-schema-interface-redundant
+    #   check-schema-iface-resource-duplicate
+    #   check-subtype-ref
+    #   check-subtype-cycle
+    #   check-duplicate-declaration
+    #   check-where-name-pattern   (entity, type, rule)
+    #   check-unique-name-pattern
+    #   check-unresolved-ref   (aggregate of eeng type/entity unparsed notes)
+    #   check-select-extended-type
+    #   check-enumeration-extended-type
+    #
+    # Severity: :error stops a clean SHTOLO run; :warning is advisory.
+    class Checker
+      Note = Struct.new(:id, :severity, :schema, :message, :node, keyword_init: true)
+
+      WHERE_LABEL = /\AWR\d+\z/i
+      UNIQUE_LABEL = /\AUR\d+\z/i
+      DECL_COLLECTIONS = %i[types entities functions procedures constants rules
+                            subtype_constraints].freeze
+
+      attr_reader :notes
+
+      def initialize(repository)
+        @repository = repository
+        @notes = []
+        # Case-folded index to LISTS of schemas: EXPRESS schema ids are
+        # case-insensitive, and two schemas differing only in case are
+        # distinct declarations that must not silently overwrite each other.
+        @by_name = {}
+        repository.schemas.compact.each do |schema|
+          next unless schema.id
+
+          (@by_name[schema.id.safe_downcase] ||= []) << schema
+        end
+      end
+
+      def check
+        @notes = []
+        @repository.schemas.compact.each { |schema| check_schema(schema) }
+        self
+      end
+
+      def errors
+        notes.select { |n| n.severity == :error }
+      end
+
+      def warnings
+        notes.select { |n| n.severity == :warning }
+      end
+
+      def valid?
+        errors.empty?
+      end
+
+      def report(io = $stdout)
+        notes.each do |n|
+          io.puts "[#{n.severity}] #{n.id}: #{n.message}"
+        end
+        io.puts "#{errors.size} error(s), #{warnings.size} warning(s)"
+        io
+      end
+
+      BUILTINS = %w[
+        integer real number string binary boolean logical generic
+        generic_entity aggregate array bag list set
+        true false unknown self const_e pi
+        abs acos asin atan cos exp format hibound hiindex length
+        log log2 log10 lobound loindex nvl odd rolesof sin sizeof
+        sqrt tan typeof usedin value value_in value_unique exists
+        blength insert remove
+      ].freeze
+
+      private
+
+      def note!(id, severity, schema, message, node = nil)
+        @notes << Note.new(id: id, severity: severity, schema: schema&.id,
+                           message: message, node: node)
+      end
+
+      def check_schema(schema)
+        check_duplicates(schema)
+        check_interfaces(schema)
+        check_self_schema_reference(schema)
+        schema.entities.each { |e| check_entity(schema, e) }
+        schema.types.each do |t|
+          check_type(schema, t)
+          check_where_rules(schema, t, t.where_rules)
+        end
+        schema.rules.each { |r| check_where_rules(schema, r, r.where_rules) }
+        check_unresolved_refs(schema)
+      end
+
+      # Self-qualified literals naming the CURRENT schema (#125): local
+      # items must not be prefixed, foreign items must name their true
+      # defining schema (ISO 10303-11 scope rules).
+      def check_self_schema_reference(schema)
+        SelfSchemaReference.matches(schema).each do |match|
+          case match.status
+          when :local
+            note!(:check_self_schema_reference, :warning, schema,
+                  "self-qualified '#{schema.id}.#{match.item}' — " \
+                  "'#{match.item}' is local; drop the prefix", match.literal)
+          when :foreign
+            note!(:check_self_schema_reference, :warning, schema,
+                  "self-qualified '#{schema.id}.#{match.item}' — item is " \
+                  "defined in '#{match.source_schema.id}'", match.literal)
+          end
+        end
+      end
+
+      # Two declarations in one schema sharing a (case-insensitive) name is
+      # invalid EXPRESS — the second can never be referenced.
+      def check_duplicates(schema)
+        seen = {}
+        DECL_COLLECTIONS.each do |coll|
+          Array(schema.public_send(coll)).each do |decl|
+            next unless decl.respond_to?(:id) && decl.id
+
+            key = decl.id.safe_downcase
+            if seen.key?(key)
+              note!(:check_duplicate_declaration, :error, schema,
+                    "duplicate declaration '#{decl.id}' (also declared in " \
+                    "#{seen[key]})", decl)
+            else
+              seen[key] = coll.to_s
+            end
+          end
+        end
+      end
+
+      # eeng: check-schema-interface-redundant +
+      #       check-schema-iface-resource-duplicate.
+      # USE FROM and REFERENCE FROM are different clauses with different
+      # semantics; only interfaces of the SAME kind are compared.
+      def check_interfaces(schema)
+        ifaces = Array(schema.interfaces)
+        ifaces.each_with_index do |iface, i|
+          target = iface_schema_name(iface)
+          next unless target
+
+          later = ifaces[(i + 1)..] || []
+          later.each do |other|
+            next unless iface_schema_name(other)&.safe_downcase == target.safe_downcase
+            next unless other.kind == iface.kind
+
+            a = resource_names(iface)
+            b = resource_names(other)
+            if a.empty? || b.empty? || (b - a).empty?
+              note!(:check_schema_interface_redundant, :warning, schema,
+                    "redundant #{iface.kind} interface to #{target}", other)
+            else
+              dup = a & b
+              next if dup.empty?
+
+              note!(:check_schema_iface_resource_duplicate, :warning, schema,
+                    "duplicate resources #{dup.join(', ')} on #{iface.kind} " \
+                    "interface to #{target}", other)
+            end
+          end
+
+          foreign = resolve_schema(target)
+          unless foreign
+            note!(:check_unresolved_ref, :error, schema,
+                  "interface schema '#{target}' not found", iface)
+            next
+          end
+
+          resource_names(iface).each do |name|
+            next if foreign_decl?(foreign, name)
+
+            note!(:check_unresolved_ref, :error, schema,
+                  "interface resource '#{name}' not in schema '#{target}'", iface)
+          end
+        end
+      end
+
+      def iface_schema_name(iface)
+        iface.schema.is_a?(String) ? iface.schema : iface.schema&.id
+      end
+
+      def resource_names(iface)
+        Array(iface.items).filter_map do |item|
+          ref = item.ref.is_a?(String) ? item.ref : item.ref&.id
+          ref || item.id
+        end.map(&:safe_downcase)
+      end
+
+      def foreign_decl?(schema, name)
+        key = name.safe_downcase
+        DECL_COLLECTIONS.any? do |coll|
+          Array(schema.public_send(coll)).any? { |d| d.id&.safe_downcase == key }
+        end
+      end
+
+      def check_entity(schema, entity)
+        Array(entity.subtype_of).each do |ref|
+          id = ref.is_a?(String) ? ref : ref.id
+          next if id.nil? || find_entity(schema, id)
+
+          note!(:check_subtype_ref, :error, schema,
+                "ENTITY #{entity.id}: subtype '#{id}' not found", entity)
+        end
+        check_subtype_cycles(schema, entity)
+
+        check_where_rules(schema, entity, entity.where_rules)
+        Array(entity.unique_rules).each do |ur|
+          next if ur.id.nil? || ur.id.match?(UNIQUE_LABEL)
+
+          note!(:check_unique_name_pattern, :warning, schema,
+                "ENTITY #{entity.id}: UNIQUE label '#{ur.id}' is not UR<n>", ur)
+        end
+      end
+
+      # Cycle detection over SUBTYPE OF edges: per-path tracking with a
+      # proven-acyclic memo, so convergent (diamond) branches are not
+      # misread as cycles (#411).
+      def check_subtype_cycles(schema, entity)
+        walk_subtype_edges(schema, entity, entity,
+                           [entity.id.safe_downcase], subtype_clean_memo(schema))
+      end
+
+      def subtype_clean_memo(schema)
+        (@subtype_clean ||= {})[schema] ||= {}
+      end
+
+      # Returns true when a cycle is reachable from +node+; +path_keys+ is
+      # the current DFS chain and +clean+ memoizes nodes proven acyclic.
+      def walk_subtype_edges(schema, start, node, path_keys, clean)
+        cyclic = false
+        Array(node.subtype_of).each do |ref|
+          id = ref.is_a?(String) ? ref : ref.id
+          next unless id
+
+          nxt = find_entity(schema, id)
+          next unless nxt
+
+          key = nxt.id.safe_downcase
+          if path_keys.include?(key)
+            note!(:check_subtype_cycle, :error, schema,
+                  "ENTITY #{start.id}: subtype inheritance cycle through " \
+                  "'#{nxt.id}'", start)
+            cyclic = true
+          elsif clean[key]
+            next
+          else
+            cyclic ||= walk_subtype_edges(
+              schema, start, nxt, path_keys + [key], clean
+            )
+          end
+        end
+        clean[node.id.safe_downcase] = true unless cyclic
+        cyclic
+      end
+
+      def check_where_rules(schema, owner, rules)
+        Array(rules).each do |wr|
+          next if wr.id.nil? || wr.id.match?(WHERE_LABEL)
+
+          note!(:check_where_name_pattern, :warning, schema,
+                "#{owner.class.name.split('::').last} #{owner.id}: " \
+                "WHERE label '#{wr.id}' is not WR<n>", wr)
+        end
+      end
+
+      def check_type(schema, type)
+        ut = type.underlying_type
+        case ut
+        when Model::DataTypes::Select
+          if ut.based_on && !resolve_type(schema, ut.based_on)
+            note!(:check_select_extended_type, :error, schema,
+                  "TYPE #{type.id}: BASED_ON '#{ref_id(ut.based_on)}' not found", type)
+          end
+        when Model::DataTypes::Enumeration
+          if ut.based_on && !resolve_type(schema, ut.based_on)
+            note!(:check_enumeration_extended_type, :error, schema,
+                  "TYPE #{type.id}: BASED_ON '#{ref_id(ut.based_on)}' not found", type)
+          end
+        end
+      end
+
+      # Walk SimpleReferences whose base_path was never filled in by the
+      # resolver — the expressir equivalent of eeng's unparsed-type notes.
+      # Locally bound identifiers are not unresolved: QUERY variables,
+      # REPEAT control variables, attribute names visible in the schema,
+      # and the `?` of aggregate bounds (#396).
+      def check_unresolved_refs(schema)
+        aliases = alias_map(schema)
+        bound = bound_identifiers(schema)
+        each_node(schema) do |node|
+          next unless node.is_a?(Model::References::SimpleReference)
+          next if node.base_path
+          next if node.parent.is_a?(Model::Declarations::InterfaceItem)
+          next if node.parent.is_a?(Model::References::AttributeReference)
+          next if node.id.nil? || builtin?(node.id)
+          next if aliases.key?(node.id.safe_downcase)
+          next if bound.include?(node.id.safe_downcase)
+
+          # Skip self-ids of declarations (entity/type names as the decl itself)
+          next if declaration_id?(schema, node)
+
+          note!(:check_unresolved_ref, :error, schema,
+                "unresolved reference '#{node.id}'", node)
+        end
+      end
+
+      # Names the schema's own constructs bind: QUERY variables, REPEAT
+      # control variables, attribute names of every entity (including the
+      # inherited ones visible through SUBTYPE OF — an over-approximation on
+      # purpose, so the error-severity walk never flags a legal identifier).
+      def bound_identifiers(schema)
+        @bound_identifiers ||= {}.compare_by_identity
+        @bound_identifiers[schema] ||= begin
+          bound = Set.new(["?"])
+          each_node(schema) do |node|
+            case node
+            when Model::Expressions::QueryExpression, Model::Statements::Repeat
+              bound << node.id.safe_downcase if node.respond_to?(:id) && node.id
+            end
+          end
+          schema.entities.each { |e| collect_attribute_names(e, bound) }
+          visible_entities(schema).each { |e| collect_attribute_names(e, bound) }
+          bound
+        end
+      end
+
+      def collect_attribute_names(entity, bound)
+        Array(entity.attributes).each do |attr|
+          bound << attr.id.safe_downcase if attr.respond_to?(:id) && attr.id
+        end
+      end
+
+      # `USE FROM s (orig AS alias)` makes `alias` the locally visible name;
+      # references written with the alias are resolved, not unresolved.
+      # Maps alias downcase => [foreign schema, original downcase].
+      def alias_map(schema)
+        @alias_maps ||= {}.compare_by_identity
+        @alias_maps[schema] ||= begin
+          map = {}
+          Array(schema.interfaces).each do |iface|
+            foreign = resolve_schema(iface_schema_name(iface))
+            next unless foreign
+
+            Array(iface.items).each do |item|
+              original = item.ref.is_a?(String) ? item.ref : item.ref&.id
+              next unless original && item.id
+
+              map[item.id.safe_downcase] = [foreign, original.safe_downcase]
+            end
+          end
+          map
+        end
+      end
+
+      def declaration_id?(_schema, node)
+        parent = node.parent
+        parent.respond_to?(:id) && parent.id&.safe_downcase == node.id.safe_downcase
+      end
+
+      def builtin?(id)
+        BUILTINS.include?(id.safe_downcase)
+      end
+
+      # Resolve a schema id case-insensitively, preferring the exact-case
+      # match when several folded ids collide (they are distinct schemas).
+      def resolve_schema(name)
+        return nil unless name
+
+        group = @by_name[name.safe_downcase]
+        return nil unless group
+
+        group.find { |s| s.id == name } || group.first
+      end
+
+      def find_entity(schema, id)
+        key = id.safe_downcase
+        own = schema.entities.find { |e| e.id.safe_downcase == key }
+        return own if own
+
+        # interface AS-renames: the alias names a foreign declaration.
+        if (aliased = alias_map(schema)[key])
+          foreign, original = aliased
+          return foreign.entities.find { |e| e.id.safe_downcase == original }
+        end
+
+        visible_entities(schema).find { |e| e.id.safe_downcase == key }
+      end
+
+      def resolve_type(schema, ref)
+        id = ref_id(ref)
+        return false unless id
+
+        key = id.safe_downcase
+        own = schema.types.find { |t| t.id.safe_downcase == key }
+        return true if own
+
+        if (aliased = alias_map(schema)[key])
+          foreign, original = aliased
+          return foreign.types.any? { |t| t.id.safe_downcase == original }
+        end
+
+        visible_types(schema).any? { |t| t.id.safe_downcase == key }
+      end
+
+      def ref_id(ref)
+        ref.is_a?(String) ? ref : ref&.id
+      end
+
+      # Foreign declarations visible through +schema+'s interfaces. With an
+      # item list, the LISTED names are AS-imported: match the originals.
+      def visible_from(schema, collection)
+        Array(schema.interfaces).flat_map do |iface|
+          foreign = resolve_schema(iface_schema_name(iface))
+          next [] unless foreign
+
+          names = resource_names(iface)
+          decls = Array(foreign.public_send(collection))
+          names.empty? ? decls : decls.select { |d| names.include?(d.id.safe_downcase) }
+        end
+      end
+
+      def visible_entities(schema)
+        visible_from(schema, :entities)
+      end
+
+      def visible_types(schema)
+        visible_from(schema, :types)
+      end
+
+      def each_node(root)
+        stack = [root]
+        until stack.empty?
+          node = stack.pop
+          yield node
+          next unless node.is_a?(Model::ModelElement)
+
+          node.class.attributes.each_key do |attr|
+            next if Model::ModelElement::SKIP_ATTRIBUTES.include?(attr)
+
+            value = node.public_send(attr)
+            case value
+            when Array then value.each { |item| stack << item if item.is_a?(Model::ModelElement) }
+            when Model::ModelElement then stack << value
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/expressir/express/concatenator.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Builds the concatenated EXPRESS artifact (expressir#247): the
+    # root schema plus every schema reachable through USE FROM /
+    # REFERENCE FROM, one section per source schema, ordered
+    # alphabetically by schema name — the Express Engine
+    # `--concat_schema` semantics.
+    module Concatenator
+      module_function
+
+      # Schemas of the interface closure of `root_schema`, deduplicated
+      # by name and sorted alphabetically — the exact set and order the
+      # concatenated artifact contains. `repository` must hold every
+      # schema the closure can reach.
+      def closure(root_schema, repository)
+        by_name = repository.schemas.to_h { |s| [s.id.safe_downcase, s] }
+        visited = {}
+        queue = [root_schema]
+        until queue.empty?
+          schema = queue.shift
+          key = schema.id.safe_downcase
+          next if visited.key?(key)
+
+          visited[key] = schema
+          Array(schema.interfaces).each do |iface|
+            target = interface_target(iface, by_name)
+            queue << target if target && !visited.key?(target.id.safe_downcase)
+          end
+        end
+        visited.values.sort_by { |s| s.id.safe_downcase }
+      end
+
+      def interface_target(iface, by_name)
+        name = iface.schema.is_a?(String) ? iface.schema : iface.schema&.id
+        return nil unless name
+
+        by_name[name.safe_downcase]
+      end
+
+      # Writes the artifact: a provenance banner, the schema index
+      # (name, file, version), then each source file verbatim behind a
+      # `-- NAME (file)` separator.
+      def write(io, schemas)
+        io.puts "(* Concatenated File produced by Expressir #{Expressir::Version::VERSION}"
+        io.puts
+        io.puts "#{schemas.size} Schemata for Concatenated File"
+        io.puts
+        schemas.each do |schema|
+          io.puts schema.id.safe_downcase
+          io.puts "   Filename: #{schema.file}"
+          io.puts "   Version:   #{schema.version&.value}" if schema.version
+          io.puts
+        end
+        io.puts " *)"
+        io.puts
+        schemas.each do |schema|
+          io.puts "--"
+          io.puts "-- #{schema.id.upcase} (#{schema.file})"
+          io.puts "--"
+          io.write(File.read(schema.file))
+          io.puts
+          io.puts
+        end
+      end
+
+      # One-call convenience: closure of the root schema found in
+      # `repository`, written to `path`.
+      def call(root_schema, repository, path)
+        schemas = closure(root_schema, repository)
+        File.open(path, "w") do |file|
+          write(file, schemas)
+        end
+        schemas
+      end
+    end
+  end
+end

data/lib/expressir/express/core.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Expressir
+  module Express
+    # Ruby bridge to the expressir-core Rust crate (parse + model
+    # extraction). Available when the native extension is built;
+    # callers fall back to the pure-Ruby parser otherwise.
+    module Core
+      NATIVE_AVAILABLE = begin
+        require "expressir/expressir_core"
+        true
+      rescue LoadError
+        begin
+          require_relative "../../../ext/expressir_core/expressir_core"
+          true
+        rescue LoadError
+          false
+        end
+      end
+
+      # Parse EXPRESS source and return the model as a plain Hash in the
+      # lutaml wire shape (to_hash-compatible), ready for
+      # Expressir::Model::ExpFile.from_hash. `path` is the relative file
+      # path used for file/base_path wiring in the model.
+      def self.parse_to_model_hash(source, path)
+        JSON.parse(::Expressir::Core.parse_to_model_hash(source, path))
+      end
+
+      # Parse EXPRESS source into the hydrated model directly: the
+      # extension walks the wire definition once and constructs every
+      # node through Serializable.instantiate (lutaml-model's fast
+      # bulk constructor). Falls back to the hash path when either
+      # the extension or the lutaml-model instantiate API is absent.
+      def self.parse_to_model(source, path)
+        unless ::Expressir::Core.respond_to?(:parse_to_model) &&
+            Model::ExpFile.respond_to?(:instantiate)
+          raise NotImplementedError,
+                "direct model construction unavailable; use parse_to_model_hash"
+        end
+
+        ::Expressir::Core.parse_to_model(source, path)
+      end
+
+      # Compile many files concurrently: jobs are [read_path,
+      # wire_path] pairs; the block receives (wire_path, model, error)
+      # per job in completion order. Workers run on native threads, so
+      # they keep compiling while the block hydrates and post-processes
+      # each model.
+      def self.parse_batch(jobs, workers: 0)
+        unless ::Expressir::Core.respond_to?(:const_defined?) &&
+            ::Expressir::Core.const_defined?(:BatchStream)
+          raise NotImplementedError,
+                "batch compile unavailable; parse files individually"
+        end
+
+        stream = ::Expressir::Core::BatchStream.start(jobs, workers)
+        while (item = stream.next)
+          path, model, error = item
+          yield(path, model, error)
+        end
+      end
+    end
+  end
+end

data/lib/expressir/express/formatter.rb

@@ -82,16 +82,38 @@ module Expressir
           return ""
         end
 
-        formatted = send(handler, node)
-        # Some handlers return nil; appending must not turn that into "".
+        place_remarks(send(handler, node), node)
+      end
+
+      private
+
+      # Puts a node's remarks back around its formatted text. Each placement
+      # is written back a different way, and this is the one place that knows
+      # all three: an inline remark is appended after the whole statement, an
+      # opener remark is spliced onto its first line, and leading remarks go
+      # on lines above it.
+      #
+      # Some handlers return nil; appending and splicing must not turn that
+      # into "".
+      def place_remarks(formatted, node)
         inline = format_inline_statement_remarks(node)
         formatted += inline if formatted.is_a?(String) && !inline.empty?
+        formatted = splice_opener_remark(formatted,
+                                         format_opener_remarks(node))
 
         leading = format_leading_statement_remarks(node)
         leading.empty? ? formatted : [*leading, formatted].join("\n")
       end
 
-      private
+      # A remark that trailed a compound statement's opening line goes back on
+      # that line. Appending it like an ordinary inline remark would carry the
+      # remark of `IF x THEN -- why` down past its END_IF.
+      def splice_opener_remark(formatted, opener)
+        return formatted unless formatted.is_a?(String) && !opener.empty?
+
+        head, newline, rest = formatted.partition("\n")
+        "#{head}#{opener}#{newline}#{rest}"
+      end
 
       def format_noop(_node)
         ""

data/lib/expressir/express/formatters/data_types_formatter.rb

@@ -190,7 +190,7 @@ module Expressir
             "GENERIC_ENTITY",
             *if node.id
                [
-                 ":",
+                 " : ",
                  node.id,
                ]
              end,
@@ -202,7 +202,7 @@ module Expressir
             "GENERIC",
             *if node.id
                [
-                 ":",
+                 " : ",
                  node.id,
                ]
              end,

data/lib/expressir/express/formatters/declarations_formatter.rb

@@ -607,6 +607,7 @@ module Expressir
             [
               "END_SCHEMA",
               ";",
+              format_end_scope_remark(node),
             ].join,
           ].join("\n")
         end