parsanol 1.2.2-aarch64-linux → 1.3.2-aarch64-linux

data/lib/parsanol/native/transformer.rb

@@ -52,40 +52,44 @@ module Parsanol
         @@symbol_cache[key] ||= key.to_sym
       end
 
+      # Symbol tags from native parser
+      SEQUENCE_SYM = :sequence
+      REPETITION_SYM = :repetition
+
       def self.transform_array(arr)
         return EMPTY_ARRAY if arr.empty? # Match Parsanol Ruby mode behavior
 
         # Check if this is a tagged array from native parser
+        # Native parser produces Symbol tags: [:sequence, item1, item2, ...]
         first = arr.first
-        if first.is_a?(String) && first.start_with?(':')
-          if first == SEQUENCE_TAG
-            # Optimized: transform items starting from index 1
-            # Avoid creating arr[1..] slice
-            len = arr.length
-            return EMPTY_ARRAY if len == 1
-
-            items = Array.new(len - 1)
-            i = 0
-            while i < len - 1
-              items[i] = transform(arr[i + 1])
-              i += 1
-            end
-            flatten_sequence(items)
-          elsif first == REPETITION_TAG
-            # Optimized: transform items starting from index 1
-            len = arr.length
-            return EMPTY_ARRAY if len == 1
-
-            items = Array.new(len - 1)
-            i = 0
-            while i < len - 1
-              items[i] = transform(arr[i + 1])
-              i += 1
-            end
-            flatten_repetition(items)
-          else
-            arr.map { |item| transform(item) }
+        if first == SEQUENCE_SYM || first == SEQUENCE_TAG
+          # Optimized: transform items starting from index 1
+          # Avoid creating arr[1..] slice
+          len = arr.length
+          return EMPTY_ARRAY if len == 1
+
+          items = Array.new(len - 1)
+          i = 0
+          while i < len - 1
+            items[i] = transform(arr[i + 1])
+            i += 1
+          end
+          flatten_sequence(items)
+        elsif first == REPETITION_SYM || first == REPETITION_TAG
+          # Optimized: transform items starting from index 1
+          len = arr.length
+          return EMPTY_ARRAY if len == 1
+
+          items = Array.new(len - 1)
+          i = 0
+          while i < len - 1
+            items[i] = transform(arr[i + 1])
+            i += 1
           end
+          flatten_repetition(items)
+        elsif first.is_a?(Symbol) || (first.is_a?(String) && first.start_with?(':'))
+          # Other tagged arrays - pass through
+          arr.map { |item| transform(item) }
         else
           # Untagged arrays from native parser are SEQUENCES
           # Apply flatten_sequence to get Parslet-compatible output
@@ -110,6 +114,9 @@ module Parsanol
         value = hash[key]
         sym_key = cached_symbol(key)
 
+        # Transform the value
+        transformed = transform(value)
+
         # Check if value is a tagged repetition from native parser
         is_tagged_repetition = value.is_a?(Array) && !value.empty? &&
                                value.first.is_a?(String) && value.first == REPETITION_TAG
@@ -124,9 +131,6 @@ module Parsanol
         # Sequences produce arrays of arrays like [[], []], not empty arrays
         is_empty_repetition = value.is_a?(Array) && value.empty?
 
-        # Transform the value
-        transformed = transform(value)
-
         # Special handling for arrays that look like character repetitions
         # (arrays of single-character Slices/strings should be joined)
         if transformed.is_a?(Array) && !transformed.empty? &&
@@ -135,7 +139,7 @@ module Parsanol
           first_slice = transformed.find { |i| i.is_a?(::Parsanol::Slice) }
           content = transformed.map { |i| slice_content(i) }.join
           transformed = if first_slice
-                          ::Parsanol::Slice.new(first_slice.offset, content, first_slice.position_cache)
+                          ::Parsanol::Slice.new(first_slice.offset, content, first_slice.input)
                         else
                           content
                         end
@@ -276,10 +280,7 @@ module Parsanol
       # Optimized: Single-pass with direct result building
       def self.flatten_sequence(items)
         return EMPTY_ARRAY if items.empty? # Match Parsanol Ruby mode
-
-        # DON'T unwrap single items - let the caller handle this
-        # This preserves repetition results like [{:x => 1}]
-        return items if items.length == 1
+        return items.first if items.length == 1
 
         # Single pass: categorize items
         merged_hash = {}
@@ -303,20 +304,33 @@ module Parsanol
             # the WHOLE sequence should be kept as array, not merged.
             if item.empty?
               # Empty repetition - skip (sequence semantics: merge rest)
+              # Don't increment total_items for empty arrays
             else
-              # Non-empty repetition - mark that we should keep as array
-              has_non_empty_array = true
-              # Still collect items for potential array result
-              item.each do |sub_item|
-                case sub_item
-                when Hash
-                  hash_count += 1
-                when ::Parsanol::Slice, String
-                  slice_or_string_parts << sub_item
+              # Check if array contains only hashes (repetition wrapper pattern)
+              # In this case, merge the inner hashes into merged_hash
+              non_hash_items = item.reject { |sub| sub.is_a?(Hash) }
+              all_items_are_hashes = non_hash_items.empty?
+
+              if all_items_are_hashes
+                # Merge all inner hashes into merged_hash
+                item.each do |sub_item|
+                  merged_hash.merge!(sub_item) if sub_item.is_a?(Hash)
                 end
+              else
+                # Non-empty repetition with non-hash items - mark that we should keep as array
+                has_non_empty_array = true
+                # Still collect items for potential array result
+                item.each do |sub_item|
+                  case sub_item
+                  when Hash
+                    hash_count += 1
+                  when ::Parsanol::Slice, String
+                    slice_or_string_parts << sub_item
+                  end
+                end
+                total_items += 1
               end
             end
-            total_items += 1
           when nil
             # Skip nil values (from lookahead or optional that didn't match)
           else
@@ -359,9 +373,9 @@ module Parsanol
         #    Result: [{:letter => "a"}, {:letter => "b"}, {:letter => "c"}]
         #
         # 3. MIXED KEYS: Hashes have DIFFERENT keys
-        #    => Keep as array
-        #    Example: [{:a => 1}, {:b => 2}]
-        #    Result: [{:a => 1}, {:b => 2}]
+        #    => Merge into single hash
+        #    Example: [{:explicitAttr => {...}}, {:whereClause => {...}}]
+        #    Result: {:explicitAttr => {...}, :whereClause => {...}}
         if hash_count == total_items && hash_count > 1
           # Check if all hashes have the same single key
           first_item = items.first
@@ -381,13 +395,66 @@ module Parsanol
 
               return items unless all_values_are_hashes
 
-              # Wrapper pattern: merge the inner hashes
-              merged_inner = {}
-              items.each do |item|
-                inner_value = item[wrapper_key]
-                merged_inner.merge!(inner_value)
+              # Check if inner hashes have the same keys or different keys
+              # REPETITION pattern (same keys like entity_decl): keep as array
+              # WRAPPER pattern (different keys like spaces vs schemaDecl): merge
+              first_inner_keys = items.first[wrapper_key].keys.to_set
+              all_same_keys = items.all? { |item| item[wrapper_key].keys.to_set == first_inner_keys }
+
+              # Check if items have single keys or multiple keys
+              # - Single key items with repeated outer key = true repetition (keep array)
+              # - Multiple key items with repeated outer key = duplicate labels in sequence (merge)
+              max_keys_per_item = items.map { |item| item.is_a?(Hash) ? item.keys.length : 0 }.max || 0
+
+              # Check if inner values are hashes with different keys
+              # This distinguishes:
+              # - True repetition: [{letter: 'a'}, {letter: 'b'}] - inner is string
+              # - Duplicate labels: [{group: {char: 'a'}}, {group: {digit: '5'}}] - inner is hash with different keys
+              inner_keys_all_same = true
+              first_inner_keys = nil
+              if items.all? { |item| item.is_a?(Hash) && item[wrapper_key].is_a?(Hash) }
+                first_inner_keys = items.first[wrapper_key].keys.to_set
+                inner_keys_all_same = items.all? { |item| item[wrapper_key].keys.to_set == first_inner_keys }
               end
-              return { wrapper_key => merged_inner }
+
+              # DUPLICATE LABELS IN SEQUENCE: multiple keys per item with repeated outer key
+              # OR inner hashes with different keys
+              # Example: [{group: {char: 'a'}}, {group: {digit: '5'}}]
+              # Ruby semantics: merge with last value wins for the outer key
+              # This is different from true repetition where each item has exactly one key
+              has_duplicate_labels = max_keys_per_item > 1 || (first_inner_keys && !inner_keys_all_same)
+
+              # Check if inner hashes have the same keys or different keys
+              first_inner_keys ||= items.first[wrapper_key].keys.to_set
+              all_same_keys = items.all? { |item| item[wrapper_key].keys.to_set == first_inner_keys }
+
+              if has_duplicate_labels
+                # DUPLICATE LABELS PATTERN: items have multiple keys with repeated outer key
+                # OR inner hashes have different keys
+                # This is a SEQUENCE with duplicate .as() labels
+                # Ruby semantics: merge and keep last value for the outer key
+                merged = {}
+                items.each do |item|
+                  item.each do |k, v|
+                    merged[k] = v  # Last value wins
+                  end
+                end
+                # Return only the wrapper key with its last value
+                return { wrapper_key => merged[wrapper_key] }
+              elsif all_same_keys
+                # TRUE REPETITION: each item has exactly one key
+                # Keep as array of hashes
+                # Example: [{letter: 'a'}, {letter: 'b'}] or [{schemaDecl: ...}, {schemaDecl: ...}]
+                return items
+              else
+                # DIFFERENT INNER KEYS with single keys: Same outer key with different inner keys
+                # This is a WRAPPER pattern - keep all items as array
+                # Example: [{:syntax => {:entityDecl => ...}}, {:syntax => {:typeDecl => ...}}]
+                # Should NOT merge or drop items - keep all declarations
+                return items
+              end
+
+              return items unless all_values_are_hashes
 
               # Repetition pattern: keep as array
 
@@ -412,10 +479,10 @@ module Parsanol
 
           if first_slice
             # Create new Slice with combined content, preserving position from first
-            ::Parsanol::Slice.new(first_slice.offset, content, first_slice.position_cache)
+            return ::Parsanol::Slice.new(first_slice.offset, content, first_slice.input)
           else
             # All plain strings (shouldn't happen with new decode_flat, but handle it)
-            slice_or_string_parts.length == 1 ? slice_or_string_parts.first : content
+            return slice_or_string_parts.length == 1 ? slice_or_string_parts.first : content
           end
         end
 
@@ -457,7 +524,7 @@ module Parsanol
 
           if first_slice
             # Create new Slice with combined content, preserving position from first
-            ::Parsanol::Slice.new(first_slice.offset, content, first_slice.position_cache)
+            ::Parsanol::Slice.new(first_slice.offset, content, first_slice.input)
           else
             # All plain strings (shouldn't happen with new decode_flat, but handle it)
             content

data/lib/parsanol/native.rb

@@ -3,223 +3,147 @@
 require 'json'
 require 'digest'
 
-# Entry point for native parsing functionality
-# Requires the individual components
 require 'parsanol/native/types'
 require 'parsanol/native/parser'
-require 'parsanol/native/transformer'
 require 'parsanol/native/serializer'
-require 'parsanol/native/dynamic'
-require 'parsanol/native/dynamic'
+require 'parsanol/native/batch_decoder'
 
 module Parsanol
   module Native
     VERSION = '0.1.0'
 
     class << self
-      # Delegate to Parser module
+      # Check if native extension is available
       def available?
         Parser.available?
       end
 
-      def parse(grammar_json, input)
-        Parser.parse(grammar_json, input)
-      end
-
-      def parse_with_grammar(root_atom, input)
-        Parser.parse_with_grammar(root_atom, input)
-      end
-
-      def parse_parslet_compatible(root_atom, input)
-        Parser.parse_parslet_compatible(root_atom, input)
-      end
-
-      def parse_batch_inputs(root_atom, inputs)
-        Parser.parse_batch_inputs(root_atom, inputs)
-      end
-
-      def parse_batch_with_transform(root_atom, inputs)
-        Parser.parse_batch_with_transform(root_atom, inputs)
-      end
-
-      def parse_raw(root_atom, input)
-        Parser.parse_raw(root_atom, input)
-      end
-
+      # Parse input with a Ruby grammar, returning clean AST with lazy line/column.
+      #
+      # Uses batch FFI format for maximum performance (3-5x faster than object-by-object).
+      # The Rust-side transformation (to_parslet_compatible) produces Parslet-compatible
+      # output that can be consumed directly by Builder.build without additional
+      # Ruby-side transformation.
+      #
+      # @param grammar [Parsanol::Atoms::Base] Ruby grammar definition
+      # @param input [String] Input string to parse
+      # @return [Hash, Array, Parsanol::Slice] Transformed AST
+      #
+      # @example Simple parsing
+      #   result = Parsanol::Native.parse(str('hello').as(:greeting), 'hello')
+      #   # => {greeting: "hello"@0}
+      #
+      # @example With lazy line/column
+      #   result = Parsanol::Native.parse(str('hello').as(:greeting), "hello\nworld")
+      #   result[:greeting].line_and_column  # => [1, 1]
+      #
+      def parse(grammar, input)
+        raise LoadError, "Native parser not available" unless available?
+
+        # Handle both grammar atoms and pre-serialized JSON strings
+        if grammar.is_a?(String)
+          grammar_json = grammar
+          grammar_atom = nil
+        else
+          grammar_json = Parser.serialize_grammar(grammar)
+          grammar_atom = grammar
+        end
+
+        # Use _parse_raw which returns properly tagged Ruby arrays via transform_ast.
+        # The batch format doesn't preserve :repetition/:sequence tags, so we use
+        # the direct FFI path. Apply the Ruby transformer to handle tags correctly.
+        raw_ast = _parse_raw(grammar_json, input)
+        BatchDecoder.decode_and_flatten(raw_ast, input, Parsanol::Slice, grammar_atom)
+      end
+
+      # Parse and return RAW AST without transformation.
+      #
+      # This returns the raw Parslet intermediate format before any transformation.
+      # Use this only if you need the raw AST for custom processing.
+      #
+      # For most use cases (including Expressir), use parse() instead which
+      # returns properly transformed AST.
+      #
+      # @param grammar [Parsanol::Atoms::Base] Ruby grammar definition
+      # @param input [String] Input string to parse
+      # @return [Hash, Array] Raw untransformed AST
+      #
+      # @example Raw parsing
+      #   result = Parsanol::Native.parse_raw(str('hello').as(:greeting), 'hello')
+      #   # => {:syntax => [{:spaces => ...}, {:greeting => "hello"@0}, {:spaces => ...}]}
+      #
+      def parse_raw(grammar, input)
+        raise LoadError, "Native parser not available" unless available?
+
+        # Handle both grammar atoms and pre-serialized JSON strings
+        if grammar.is_a?(String)
+          grammar_json = grammar
+        else
+          grammar_json = Parser.serialize_grammar(grammar)
+        end
+
+        # Use batch_raw format for raw AST (no transformation)
+        slice_class = Parsanol::Slice
+        batch_data = _parse_batch_raw(grammar_json, input)
+
+        # Decode without transformation - raw AST format
+        BatchDecoder.decode(batch_data, input, slice_class)
+      end
+
+      # Fast batch parsing - uses u64 array format to minimize FFI overhead.
+      #
+      # This is 3-5x faster than regular parse() for large grammars.
+      # The batch format passes a flat u64 array across FFI, then decodes
+      # in pure Ruby, avoiding expensive per-node FFI calls.
+      #
+      # Returns RAW AST without transformation. For Expressir use case,
+      # use parse() instead which returns properly transformed AST.
+      #
+      # @param grammar_json [String] Pre-serialized grammar JSON
+      # @param input [String] Input string to parse
+      # @param slice_class [Class] The Slice class to use for string refs
+      # @return [Hash, Array, Slice] Raw AST (not transformed)
+      def parse_batch(grammar_json, input, slice_class)
+        raise LoadError, "Native parser not available" unless available?
+
+        # Call native extension's _parse_batch_raw method (named with _raw suffix
+        # to avoid conflict with this Ruby wrapper method)
+        batch_data = _parse_batch_raw(grammar_json, input)
+        BatchDecoder.decode(batch_data, input, slice_class)
+      end
+
+      # Get the Slice class
+      private def get_slice_class
+        Parsanol::Slice
+      end
+
+      # Serialize a Ruby grammar to JSON (cached).
+      #
+      # @param root_atom [Parsanol::Atoms::Base] Root atom of the grammar
+      # @return [String] JSON string
       def serialize_grammar(root_atom)
         Parser.serialize_grammar(root_atom)
       end
 
+      # Clear grammar caches (call if grammar changes)
       def clear_cache
         Parser.clear_cache
       end
 
+      # Get cache statistics
       def cache_stats
         Parser.cache_stats
       end
-
-      # Serialized Mode (JSON Output)
-      def parse_to_json(grammar_json, input)
-        Parser.parse_to_json(grammar_json, input)
-      end
-
-      # ZeroCopy Mode (Direct Ruby Objects)
-      def parse_to_objects(grammar_json, input, type_map = nil)
-        Parser.parse_to_objects(grammar_json, input, type_map)
-      end
-
-      def convert_slices(obj, input)
-        Parser.convert_slices(obj, input)
-      end
-
-      # Source Location Tracking
-      def parse_with_spans(grammar_json, input)
-        Parser.parse_with_spans(grammar_json, input)
-      end
-
-      def get_span(result, node_id)
-        Parser.get_span(result, node_id)
-      end
-
-      # Grammar Composition
-      def grammar_import(builder_json, grammar_json, prefix = nil)
-        Parser.grammar_import(builder_json, grammar_json, prefix)
-      end
-
-      def grammar_rule_mut(builder_json, rule_name)
-        Parser.grammar_rule_mut(builder_json, rule_name)
-      end
-
-      # Streaming Parser
-      def streaming_parser_new(grammar_json)
-        Parser.streaming_parser_new(grammar_json)
-      end
-
-      def streaming_parser_add_chunk(parser, chunk)
-        Parser.streaming_parser_add_chunk(parser, chunk)
-      end
-
-      def streaming_parser_parse_chunk(parser)
-        Parser.streaming_parser_parse_chunk(parser)
-      end
-
-      # Incremental Parser
-      def incremental_parser_new(grammar_json, initial_input)
-        Parser.incremental_parser_new(grammar_json, initial_input)
-      end
-
-      def incremental_parser_apply_edit(parser, start, deleted, inserted = '')
-        Parser.incremental_parser_apply_edit(parser, start, deleted, inserted)
-      end
-
-      def incremental_parser_reparse(parser, new_input = nil)
-        Parser.incremental_parser_reparse(parser, new_input)
-      end
-
-      # Streaming Builder - uses native parse_with_builder directly (exposed from Rust)
-      # The native function is exposed directly on Parsanol::Native module
-
-      # Alias for parse_with_builder (same functionality)
-      def parse_with_callback(grammar_json, input, callback)
-        parse_with_builder(grammar_json, input, callback)
-      end
-
-      # Parallel Parsing - uses native _parse_batch_parallel
-      def parse_batch_parallel(grammar_json, inputs, num_threads: nil)
-        _parse_batch_parallel(grammar_json, inputs, num_threads || 0)
-      end
-
-      # Security / Limits - uses native _parse_with_limits
-      def parse_with_limits(grammar_json, input, max_input_size: 100 * 1024 * 1024, max_recursion_depth: 1000)
-        _parse_with_limits(grammar_json, input, max_input_size, max_recursion_depth)
-      end
-
-      # Debug Tools
-      def parse_with_trace(grammar_json, input)
-        Parser.parse_with_trace(grammar_json, input)
-      end
-
-      def grammar_to_mermaid(grammar_json)
-        Parser.grammar_to_mermaid(grammar_json)
-      end
-
-      def grammar_to_dot(grammar_json)
-        Parser.grammar_to_dot(grammar_json)
-      end
-
-      # Legacy internal methods (for backward compatibility)
-      def _parse_with_spans(grammar_json, input)
-        Parser.send(:_parse_with_spans, grammar_json, input)
-      end
-
-      def _get_span(result, node_id)
-        Parser.send(:_get_span, result, node_id)
-      end
-
-      def _grammar_import(builder_json, grammar_json, prefix)
-        Parser.send(:_grammar_import, builder_json, grammar_json, prefix)
-      end
-
-      def _grammar_rule_mut(builder_json, rule_name)
-        Parser.send(:_grammar_rule_mut, builder_json, rule_name)
-      end
-
-      def _streaming_parser_new(grammar_json)
-        Parser.send(:_streaming_parser_new, grammar_json)
-      end
-
-      def _streaming_parser_add_chunk(parser, chunk)
-        Parser.send(:_streaming_parser_add_chunk, parser, chunk)
-      end
-
-      def _streaming_parser_parse_chunk(parser)
-        Parser.send(:_streaming_parser_parse_chunk, parser)
-      end
-
-      def _incremental_parser_new(grammar_json, initial_input)
-        Parser.send(:_incremental_parser_new, grammar_json, initial_input)
-      end
-
-      def _incremental_parser_apply_edit(parser, start, deleted, inserted)
-        Parser.send(:_incremental_parser_apply_edit, parser, start, deleted, inserted)
-      end
-
-      def _incremental_parser_reparse(parser, new_input)
-        Parser.send(:_incremental_parser_reparse, parser, new_input)
-      end
-
-      def _parse_batch_parallel(grammar_json, inputs, num_threads)
-        Parser.send(:_parse_batch_parallel, grammar_json, inputs, num_threads)
-      end
-
-      def _parse_with_limits(grammar_json, input, max_input_size, max_recursion_depth)
-        Parser.send(:_parse_with_limits, grammar_json, input, max_input_size, max_recursion_depth)
-      end
-
-      def _parse_with_trace(grammar_json, input)
-        Parser.send(:_parse_with_trace, grammar_json, input)
-      end
-
-      def _grammar_to_mermaid(grammar_json)
-        Parser.send(:_grammar_to_mermaid, grammar_json)
-      end
-
-      def _grammar_to_dot(grammar_json)
-        Parser.send(:_grammar_to_dot, grammar_json)
-      end
     end
   end
 end
 
 # Attempt to load native extension
-# rb_sys puts native extensions in version-specific directories (e.g., parsanol/3.2/parsanol_native)
 begin
-  # Try version-specific path first (for precompiled gems)
   ruby_version = RUBY_VERSION.split('.').take(2).join('.')
   require "parsanol/#{ruby_version}/parsanol_native"
 rescue LoadError
   begin
-    # Fall back to generic path (for locally compiled extensions)
     require 'parsanol/parsanol_native'
   rescue LoadError
     # Native extension not built yet

data/lib/parsanol/parser.rb

@@ -168,12 +168,8 @@ module Parsanol
     #
     def parse_native(input, opts)
       if Parsanol::Native.available?
-        # Build line cache for position info
-        line_cache = Parsanol::Source::LineCache.new
-        line_cache.scan_for_line_endings(0, input)
-
-        # Parse with position info (now the default)
-        Parsanol::Native.parse_parslet_compatible(root, input, line_cache)
+        # Parse with lazy line/column support
+        Parsanol::Native.parse(root, input)
       else
         parse_ruby(input, opts)
       end