expressir 2.4.0 → 2.4.2

data/lib/expressir/express/builders/expression_builder.rb

@@ -10,9 +10,9 @@ module Expressir
         # Expression
         def build_expression(ast_data)
           left = if ast_data[:simple_expression]
-                   Builder.build_simple_expression(ast_data[:simple_expression])
+                   Builder.build_node(:simple_expression, ast_data[:simple_expression])
                  elsif ast_data[:logical_expression]
-                   Builder.build_simple_expression(ast_data[:logical_expression][:simple_expression])
+                   Builder.build_node(:simple_expression, ast_data[:logical_expression][:simple_expression])
                  end
 
           if ast_data[:rel_op_extended] && ast_data[:rhs]
@@ -29,18 +29,18 @@ module Expressir
         end
 
         def build_logical_expression(ast_data)
-          Builder.build_simple_expression(ast_data[:simple_expression])
+          Builder.build_node(:simple_expression, ast_data[:simple_expression])
         end
 
         def build_numeric_expression(ast_data)
-          Builder.build_simple_expression(ast_data[:simple_expression])
+          Builder.build_node(:simple_expression, ast_data[:simple_expression])
         end
 
         # Simple expression (addition/subtraction chain)
         def build_simple_expression(ast_data)
           return nil unless ast_data[:term]
 
-          term = Builder.build_term(ast_data[:term])
+          term = Builder.build_node(:term, ast_data[:term])
           rhs = ast_data[:rhs]
 
           return term if rhs.nil? || (rhs.is_a?(Array) && rhs.empty?)
@@ -60,7 +60,7 @@ module Expressir
             op_data = item[:operator]
             operators << extract_operator(op_data[:add_like_op]) if op_data
             if item[:term]
-              operands << Builder.build_term(item[:term])
+              operands << Builder.build_node(:term, item[:term])
             end
           end
 
@@ -71,7 +71,7 @@ module Expressir
         def build_term(ast_data)
           return nil unless ast_data[:factor]
 
-          factor = Builder.build_factor(ast_data[:factor])
+          factor = Builder.build_node(:factor, ast_data[:factor])
           rhs = ast_data[:rhs]
 
           return factor if rhs.nil? || (rhs.is_a?(Array) && rhs.empty?)
@@ -90,7 +90,7 @@ module Expressir
             item = r[:item] || r
             op_data = item[:multiplication_like_op] || item[:mul_like_op]
             operators << extract_operator(op_data) if op_data
-            operands << Builder.build_factor(item[:factor]) if item[:factor]
+            operands << Builder.build_node(:factor, item[:factor]) if item[:factor]
           end
 
           build_binary_expression(operands, operators)
@@ -99,14 +99,14 @@ module Expressir
         def build_factor(ast_data)
           return nil unless ast_data[:simple_factor]
 
-          Builder.build_simple_factor(ast_data[:simple_factor])
+          Builder.build_node(:simple_factor, ast_data[:simple_factor])
         end
 
         def build_simple_factor(ast_data)
           return nil unless ast_data
 
           if ast_data[:primary]
-            Builder.build_primary(ast_data[:primary])
+            Builder.build_node(:primary, ast_data[:primary])
           elsif ast_data[:simple_factor_expression]
             Builder.build_node(:simple_factor_expression,
                                ast_data[:simple_factor_expression])
@@ -116,7 +116,7 @@ module Expressir
           elsif ast_data[:constant_factor]
             Builder.build_node(:constant_factor, ast_data[:constant_factor])
           elsif ast_data[:expression]
-            Builder.build_expression(ast_data[:expression])
+            Builder.build_node(:expression, ast_data[:expression])
           elsif ast_data[:aggregate_initializer]
             Builder.build_node(:aggregate_initializer,
                                ast_data[:aggregate_initializer])
@@ -137,21 +137,21 @@ module Expressir
 
         def build_simple_factor_expression(ast_data)
           if ast_data[:primary]
-            Builder.build_primary(ast_data[:primary])
+            Builder.build_node(:primary, ast_data[:primary])
           elsif ast_data[:expression]
-            Builder.build_expression(ast_data[:expression])
+            Builder.build_node(:expression, ast_data[:expression])
           end
         end
 
         def build_simple_factor_unary_expression(ast_data)
           op = extract_unary_op(ast_data[:unary_op])
           operand = if ast_data[:simple_factor]
-                      Builder.build_simple_factor(ast_data[:simple_factor])
+                      Builder.build_node(:simple_factor, ast_data[:simple_factor])
                     elsif ast_data[:simple_factor_expression]
                       Builder.build_node(:simple_factor_expression,
                                          ast_data[:simple_factor_expression])
                     elsif ast_data[:primary]
-                      Builder.build_primary(ast_data[:primary])
+                      Builder.build_node(:primary, ast_data[:primary])
                     end
 
           if op
@@ -181,7 +181,7 @@ module Expressir
           elsif ast_data[:population]
             build_population(ast_data[:population], ast_data[:qualifier])
           elsif ast_data[:expression]
-            Builder.build_expression(ast_data[:expression])
+            Builder.build_node(:expression, ast_data[:expression])
           end
         end
 

data/lib/expressir/express/cache.rb

@@ -1,8 +1,17 @@
+require "digest"
 require "zlib"
 
 module Expressir
   module Express
     class Cache
+      # Format: magic + SHA-256 (raw, 32 bytes) + Zlib-deflated Marshal.dump
+      # of Model::Cache. The digest detects corruption; a corrupted cache is
+      # disposable — callers drop the file and regenerate. Marshal is only
+      # safe for trusted, locally generated cache files.
+      MAGIC = "EXPRC1".freeze
+
+      DIGEST_BYTES = Digest::SHA256.digest("").bytesize
+
       # Save Express model into a cache file
       # @param file [String] cache file path
       # @param content [Model::ModelElement] Express model
@@ -11,7 +20,7 @@ module Expressir
       # @return [nil]
       def self.to_file(file, content, root_path: nil,
 test_overwrite_version: nil)
-        version = test_overwrite_version || VERSION
+        version = test_overwrite_version || Expressir::Version::VERSION
 
         cache = Model::Cache.new(
           version: version,
@@ -19,10 +28,9 @@ test_overwrite_version: nil)
           root_path: root_path,
         )
 
-        yaml = cache.to_yaml
-        yaml_compressed = Zlib::Deflate.deflate(yaml)
+        data = Zlib::Deflate.deflate(Marshal.dump(cache), 1)
+        File.binwrite(file, "#{MAGIC}#{Digest::SHA256.digest(data)}#{data}")
 
-        File.binwrite(file, yaml_compressed)
         nil
       end
 
@@ -31,18 +39,37 @@ test_overwrite_version: nil)
       # @param root_path [String] Express repository root path, to be prepended to Express file paths if loading a portable cache file
       # @param test_overwrite_version [String] don't use, only for tests
       # @return [Model::ModelElement] Express model
+      # @raise [Error::CacheCorruptedError] if the file is not a valid cache file or its digest does not match
+      # @raise [Error::CacheVersionMismatchError] if the cache was written by another Expressir version
       def self.from_file(file, root_path: nil, test_overwrite_version: nil)
-        version = test_overwrite_version || VERSION
+        version = test_overwrite_version || Expressir::Version::VERSION
 
-        yaml_compressed = File.binread(file)
-        yaml = Zlib::Inflate.inflate(yaml_compressed)
-        cache = Model::Cache.from_yaml(yaml)
+        raw = File.binread(file)
+        data = Zlib::Inflate.inflate(validated_data(raw))
+        cache = Marshal.load(data) # rubocop:disable Security/MarshalLoad
 
         if cache.version != version
           raise Error::CacheVersionMismatchError.new(cache.version, version)
         end
 
         cache
+      rescue TypeError, ArgumentError, RangeError
+        raise Error::CacheCorruptedError
+      end
+
+      def self.validated_data(raw)
+        header = MAGIC.bytesize + DIGEST_BYTES
+        unless raw.bytesize > header && raw.start_with?(MAGIC)
+          raise Error::CacheCorruptedError
+        end
+
+        digest = raw.byteslice(MAGIC.bytesize, DIGEST_BYTES)
+        data = raw.byteslice(header, raw.bytesize - header)
+        unless Digest::SHA256.digest(data) == digest
+          raise Error::CacheCorruptedError
+        end
+
+        data
       end
     end
   end

data/lib/expressir/express/error.rb

@@ -38,6 +38,30 @@ module Expressir
         end
       end
 
+      # Error raised when a cache file is not a valid cache file or fails integrity validation
+      class CacheCorruptedError < ExpressError
+        def initialize(message = "Cache file is corrupted or not a valid Expressir cache file")
+          super
+        end
+      end
+
+      # Error raised when parallel file parsing loses a worker or its data
+      class ParallelParseError < ExpressError
+        def initialize(message = "Parallel parsing failed")
+          super
+        end
+      end
+
+      # Error raised when streaming parsing is requested but unavailable.
+      # The streaming paths require a parsanol release whose
+      # parse_with_builder is stable; parse_fresh cannot parse EXPRESS
+      # without packrat memoization (see parsanol-ruby#52).
+      class StreamingUnsupportedError < ExpressError
+        def initialize(message = "Streaming parsing is not supported by the installed parsanol release")
+          super
+        end
+      end
+
       # Base class for visitor-related errors
       class VisitorError < ExpressError; end
 

data/lib/expressir/express/formatter.rb

@@ -9,6 +9,15 @@ module Expressir
       end
 
       def self.register_formatter(model_class, method_name)
+        # Guard: fail at registration time if the method doesn't exist on
+        # this class (or one of its included modules). Includes private
+        # methods since format handlers are private (TODO.bugs/19).
+        unless method_defined?(method_name) || private_method_defined?(method_name)
+          raise ArgumentError,
+                "Formatter.register_formatter: method :#{method_name} " \
+                "is not defined for #{model_class}"
+        end
+
         @format_registry[model_class] = method_name
       end
 
@@ -23,14 +32,6 @@ module Expressir
       include Formatters::DataTypesFormatter
       include Formatters::DeclarationsFormatter
 
-      # Handlers for types implemented directly in this class
-      register_formatter Model::Repository, :format_repository
-      register_formatter Model::ExpFile, :format_exp_file
-      register_formatter Model::Declarations::SchemaVersionItem, :format_noop
-      register_formatter Model::Declarations::InterfacedItem, :format_noop
-      register_formatter Model::Cache, :format_noop
-      register_formatter Model::ModelElement, :format_noop
-
       INDENT_CHAR = " ".freeze
       INDENT_WIDTH = 2
       INDENT = INDENT_CHAR * INDENT_WIDTH
@@ -62,12 +63,6 @@ module Expressir
         Model::SupertypeExpressions::BinarySupertypeExpression::ANDOR => 2,
       }.freeze
 
-      private_constant :INDENT_CHAR
-      private_constant :INDENT_WIDTH
-      private_constant :INDENT
-      private_constant :OPERATOR_PRECEDENCE
-      private_constant :SUPERTYPE_OPERATOR_PRECEDENCE
-
       attr_accessor :no_remarks
 
       def initialize(no_remarks: false)
@@ -82,12 +77,22 @@ module Expressir
         return "" if node.nil?
 
         handler = self.class.format_registry[node.class]
-        return public_send(handler, node) if handler
-
-        warn "#{node.class.name} format not implemented"
-        ""
+        unless handler
+          warn "#{node.class.name} format not implemented"
+          return ""
+        end
+
+        formatted = send(handler, node)
+        # Some handlers return nil; appending must not turn that into "".
+        inline = format_inline_statement_remarks(node)
+        formatted += inline if formatted.is_a?(String) && !inline.empty?
+
+        leading = format_leading_statement_remarks(node)
+        leading.empty? ? formatted : [*leading, formatted].join("\n")
       end
 
+      private
+
       def format_noop(_node)
         ""
       end
@@ -107,6 +112,16 @@ module Expressir
 
         str.split("\n").map { |x| "#{INDENT}#{x}" }.join("\n")
       end
+
+      # Handlers for types implemented directly in this class.
+      # Registered at the bottom so the methods exist when the
+      # registration-time method_defined? guard runs.
+      register_formatter Model::Repository, :format_repository
+      register_formatter Model::ExpFile, :format_exp_file
+      register_formatter Model::Declarations::SchemaVersionItem, :format_noop
+      register_formatter Model::Declarations::InterfacedItem, :format_noop
+      register_formatter Model::Cache, :format_noop
+      register_formatter Model::ModelElement, :format_noop
     end
   end
 end

data/lib/expressir/express/formatters/declarations_formatter.rb

@@ -292,6 +292,7 @@ module Expressir
             *if node.statements&.length&.positive?
                indent(node.statements.map { |x| format(x) }.join("\n"))
              end,
+            *format_trailing_region_remarks(node, :statements).map { |x| indent(x) },
             [
               "END_FUNCTION",
               ";",
@@ -413,6 +414,7 @@ module Expressir
             *if node.statements&.length&.positive?
                indent(node.statements.map { |x| format(x) }.join("\n"))
              end,
+            *format_trailing_region_remarks(node, :statements).map { |x| indent(x) },
             [
               "END_PROCEDURE",
               ";",
@@ -421,14 +423,29 @@ module Expressir
           ].join("\n")
         end
 
+        # True when the statement holds a remark that only it can render.
+        # Under no_remarks nothing is emitted anyway, so the statement stays
+        # filtered and remark-free output is byte-identical to before.
+        def carries_placed_remark?(statement)
+          return false if @no_remarks
+
+          Array(statement.untagged_remarks).any?(&:placement)
+        end
+
         def format_declarations_rule(node)
           # Filter out statements that only exist to hold remarks in rules
           # (ALIAS/REPEAT with only Null sub-statements, query assignments, or Null statements)
           formatted_statements = []
           if node.statements&.length&.positive?
             formatted_statements = node.statements.reject do |stmt|
+              # A statement holding a placed remark is the only thing that can
+              # render it, so filtering it would silently drop the comment —
+              # and PrettyFormatter, which does not filter, would then produce
+              # different content from this formatter.
+              if carries_placed_remark?(stmt)
+                false
               # Filter ALIAS/REPEAT with only Null statements
-              if stmt.is_a?(Model::Statements::Alias) || stmt.is_a?(Model::Statements::Repeat)
+              elsif stmt.is_a?(Model::Statements::Alias) || stmt.is_a?(Model::Statements::Repeat)
                 stmt.statements&.all?(Model::Statements::Null)
               # Filter query assignments (assignments with QueryExpression that exist only for remarks)
               elsif stmt.is_a?(Model::Statements::Assignment) &&
@@ -494,12 +511,17 @@ module Expressir
             *if formatted_statements.length.positive?
                indent(formatted_statements.map { |x| format(x) }.join("\n"))
              end,
+            # A RULE's executable body ends at WHERE, so its trailing
+            # remarks belong above that keyword, not after the where rules.
+            *format_trailing_region_remarks(node, :statements).map { |x| indent(x) },
             *if node.where_rules&.length&.positive?
                [
                  "WHERE",
                  indent(node.where_rules.map { |x| format(x) }.join("\n")),
                ]
              end,
+            # A comment after the last where rule closes the WHERE section.
+            *format_trailing_region_remarks(node, :where_rules).map { |x| indent(x) },
             [
               "END_RULE",
               ";",
@@ -525,7 +547,7 @@ module Expressir
             *if node.interfaces&.length&.positive?
                [
                  "",
-                 node.interfaces.map { |x| format(x) }.join("\n"),
+                 node.interfaces.map { |x| format(x) }.join("\n\n"),
                ]
              end,
           ].join("\n")

data/lib/expressir/express/formatters/remark_formatter.rb

@@ -68,9 +68,11 @@ module Expressir
         def format_end_scope_remark(node)
           return "" if @no_remarks
           return "" unless node.is_a?(Model::ModelElement)
-          return "" if node.untagged_remarks.nil? || node.untagged_remarks.empty?
 
-          remark = node.untagged_remarks.last
+          # Only legacy (unplaced) remarks belong after the END_* keyword.
+          # Remarks carrying an explicit placement are rendered by the
+          # placement-aware emitters, and must not be emitted twice.
+          remark = Array(node.untagged_remarks).reject(&:placement).last
           return "" unless remark.is_a?(Model::RemarkInfo)
 
           text = remark.text
@@ -136,6 +138,77 @@ module Expressir
           end.join("\n")
         end
 
+        # Remarks that close one body of `node`, emitted just before that
+        # body's closing keyword. Only explicit TRAILING remarks for the named
+        # region qualify: legacy (nil) remarks keep their historical emission
+        # path so old caches render exactly as they did before.
+        def format_trailing_region_remarks(node, region)
+          return [] if @no_remarks
+          return [] unless node.is_a?(Model::ModelElement)
+
+          Array(node.untagged_remarks).filter_map do |remark|
+            next unless remark.trailing_region?(region)
+
+            formatted = format_untagged_remark(remark)
+            formatted unless formatted.empty?
+          end
+        end
+
+        # Remarks written after their statement on the same line. Attachment
+        # only assigns these to single-line statements, so appending keeps
+        # them on that statement's line.
+        def format_inline_statement_remarks(node)
+          return "" if @no_remarks
+          return "" unless node.is_a?(Model::Statement)
+
+          Array(node.untagged_remarks).filter_map do |remark|
+            next unless remark.inline?
+
+            formatted = format_untagged_remark(remark)
+            formatted unless formatted.empty?
+          end.map { |text| " #{text}" }.join
+        end
+
+        def format_leading_statement_remarks(node)
+          return [] if @no_remarks
+          return [] unless node.is_a?(Model::Statement)
+
+          Array(node.untagged_remarks).filter_map do |remark|
+            next unless remark.leading?
+
+            formatted = format_untagged_remark(remark)
+            formatted unless formatted.empty?
+          end
+        end
+
+        # Block-end emission for ALIAS/REPEAT bodies. Tagged remarks are
+        # stored as bare text in node.remarks alongside a mirror of every
+        # untagged text (add_remark dual-store), so "genuinely tagged" is
+        # derivable only by multiset-subtracting the untagged texts.
+        def format_block_end_remarks(node)
+          return [] if @no_remarks
+
+          untagged = Array(node.untagged_remarks)
+          remaining = untagged.map(&:text).tally
+          tagged = Array(node.remarks).compact.reject do |text|
+            next false unless remaining[text]&.positive?
+
+            remaining[text] -= 1
+            true
+          end
+
+          [
+            *tagged.map { |text| format_remark(node, text) },
+            # Leading and inline remarks are emitted by their own paths;
+            # including them here would render them a second time.
+            *untagged.reject { |r| r.leading? || r.inline? }
+              .filter_map do |remark|
+                formatted = format_untagged_remark(remark)
+                formatted unless formatted.empty?
+              end,
+          ]
+        end
+
         def format_remarks(node)
           remarks = []
 

data/lib/expressir/express/formatters/statements_formatter.rb

@@ -43,7 +43,7 @@ module Expressir
             *if node.statements&.length&.positive?
                indent(node.statements.map { |x| format(x) }.join("\n"))
              end,
-            *format_remarks(node),
+            *format_block_end_remarks(node).map { |x| indent(x) },
             [
               "END_ALIAS",
               ";",
@@ -88,6 +88,10 @@ module Expressir
             *if node.actions&.length&.positive?
                node.actions.map { |x| format(x) }
              end,
+            # Closes the last action: before OTHERWISE when one exists,
+            # otherwise before END_CASE.
+            *format_trailing_region_remarks(node, :action_statements)
+              .map { |x| indent(x) },
             *if node.otherwise_statement
                [
                  [
@@ -98,6 +102,8 @@ module Expressir
                  indent(format(node.otherwise_statement)),
                ]
              end,
+            *format_trailing_region_remarks(node, :otherwise_statements)
+              .map { |x| indent(x) },
             [
               "END_CASE",
               ";",
@@ -123,6 +129,7 @@ module Expressir
             *if statements.length.positive?
                indent(statements.map { |x| format(x) }.join("\n"))
              end,
+            *format_trailing_region_remarks(node, :statements).map { |x| indent(x) },
             [
               "END",
               ";",
@@ -138,6 +145,14 @@ module Expressir
         end
 
         def format_statements_if(node)
+          has_else = node.else_statements&.length&.positive?
+          # A comment closing the THEN body sits before ELSE when there is
+          # one, and before END_IF otherwise.
+          then_trailing = format_trailing_region_remarks(node, :statements)
+            .map { |x| indent(x) }
+          else_trailing = format_trailing_region_remarks(node, :else_statements)
+            .map { |x| indent(x) }
+
           [
             [
               "IF",
@@ -149,12 +164,14 @@ module Expressir
             *if node.statements&.length&.positive?
                indent(node.statements.map { |x| format(x) }.join("\n"))
              end,
-            *if node.else_statements&.length&.positive?
+            *(then_trailing if has_else),
+            *if has_else
                [
                  "ELSE",
                  indent(node.else_statements.map { |x| format(x) }.join("\n")),
                ].join("\n")
              end,
+            *(has_else ? else_trailing : then_trailing),
             [
               "END_IF",
               ";",
@@ -214,7 +231,7 @@ module Expressir
             *if statements.length.positive?
                indent(statements.map { |x| format(x) }.join("\n"))
              end,
-            *format_remarks(node),
+            *format_block_end_remarks(node).map { |x| indent(x) },
             [
               "END_REPEAT",
               ";",