parsanol 1.0.1-aarch64-linux → 1.0.2-aarch64-linux

data/lib/parsanol/native/transformer.rb

@@ -5,15 +5,15 @@ module Parsanol
     # Transforms native AST format to Parslet-compatible format
     #
     # Native format from Rust parser:
-    #   - Strings: "text"
+    #   - Slices: Parsanol::Slice objects (with position info)
     #   - Sequences: [":sequence", item1, item2, ...]
     #   - Repetitions: [":repetition", item1, item2, ...]
     #   - Named captures: {"name" => value}
     #
     # Parslet format:
-    #   - Strings: "text" (with Parsanol::Slice for position info)
+    #   - Slices: Preserved as Parsanol::Slice (with position info)
     #   - Sequences: merged hash {:key1 => val1, :key2 => val2, ...}
-    #   - Repetitions: array of items (or "" if empty string-like)
+    #   - Repetitions: array of items (or joined Slices if all are Slices)
     #   - Named wrapping Repetition: {:name => [{:name => item1}, {:name => item2}, ...]}
     #
     class AstTransformer
@@ -128,10 +128,17 @@ module Parsanol
         transformed = transform(value)
 
         # Special handling for arrays that look like character repetitions
-        # (arrays of single-character strings should be joined)
+        # (arrays of single-character Slices/strings should be joined)
         if transformed.is_a?(Array) && !transformed.empty? &&
-           transformed.all? { |item| item.is_a?(String) && item.length == 1 }
-          transformed = transformed.join
+           transformed.all? { |item| slice_or_string?(item) && item_length(item) == 1 }
+          # Join preserving position from first Slice
+          first_slice = transformed.find { |i| i.is_a?(::Parsanol::Slice) }
+          content = transformed.map { |i| slice_content(i) }.join
+          transformed = if first_slice
+                          ::Parsanol::Slice.new(first_slice.offset, content, first_slice.position_cache)
+                        else
+                          content
+                        end
         end
 
         # Check for UNTAGGED repetition pattern (native output):
@@ -151,11 +158,21 @@ module Parsanol
         elsif transformed.is_a?(Array)
           transform_array_value(sym_key, transformed)
         else
-          # Simple value (string, nil, etc.) - most common case
+          # Simple value (Slice, string, nil, etc.) - most common case
           { sym_key => transformed }
         end
       end
 
+      # Get content from Slice or string
+      def self.slice_content(value)
+        value.is_a?(::Parsanol::Slice) ? value.content : value.to_s
+      end
+
+      # Get length of Slice or string
+      def self.item_length(value)
+        value.is_a?(::Parsanol::Slice) ? value.length : value.to_s.length
+      end
+
       # Handle repetition values (named wrapping repetition)
       def self.transform_repetition_value(sym_key, transformed)
         if transformed.is_a?(Array)
@@ -169,7 +186,9 @@ module Parsanol
             # Wrap each item with the name
             { sym_key => transformed.map { |item| { sym_key => item } } }
           end
-        elsif transformed == EMPTY_STRING
+        elsif transformed.is_a?(::Parsanol::Slice) && transformed.empty?
+          { sym_key => EMPTY_ARRAY } # Empty repetition should be [], not empty Slice
+        elsif transformed.is_a?(String) && transformed == EMPTY_STRING
           { sym_key => EMPTY_ARRAY } # Empty repetition should be [], not ""
         else
           { sym_key => transformed }
@@ -180,10 +199,10 @@ module Parsanol
       def self.transform_array_value(sym_key, transformed)
         if transformed.empty?
           # For empty arrays, we need to determine if this is a repetition or sequence
-          # Repetitions should return [], sequences should return ""
-          # We can't tell from the value alone, so we return "" (sequence semantics)
+          # Repetitions should return [], sequences should return empty Slice
+          # We can't tell from the value alone, so we return empty Slice (sequence semantics)
           # The repetition detection in transform_single_key_hash will handle the other case
-          { sym_key => EMPTY_STRING }
+          { sym_key => ::Parsanol::Slice.new(0, EMPTY_STRING, nil) }
         elsif transformed.all? { |v| v.is_a?(Hash) && v.keys.length == 1 && v.key?(sym_key) }
           # Items already have the parent key (repetition pattern) - keep as-is
           { sym_key => transformed }
@@ -242,8 +261,8 @@ module Parsanol
 
       # Flatten sequence items according to Parslet semantics:
       # 1. If ALL items are hashes, return as array (this is a repetition result)
-      # 2. If there are named captures (hashes) among strings, return ONLY the merged hash (discard strings)
-      # 3. If only strings, join them (or return single string)
+      # 2. If there are named captures (hashes) among Slices/strings, return ONLY the merged hash (discard Slices/strings)
+      # 3. If only Slices/strings, join them preserving position from first Slice
       # 4. Return single value if only one item
       #
       # This matches Parslet's behavior where:
@@ -264,7 +283,7 @@ module Parsanol
 
         # Single pass: categorize items
         merged_hash = {}
-        string_parts = []
+        slice_or_string_parts = []
         hash_count = 0
         total_items = 0
         has_non_empty_array = false
@@ -275,8 +294,8 @@ module Parsanol
             merged_hash.merge!(item)
             hash_count += 1
             total_items += 1
-          when String
-            string_parts << item
+          when ::Parsanol::Slice, String
+            slice_or_string_parts << item
             total_items += 1
           when Array
             # Check if this is a non-empty array (repetition result with content)
@@ -292,8 +311,8 @@ module Parsanol
                 case sub_item
                 when Hash
                   hash_count += 1
-                when String
-                  string_parts << sub_item
+                when ::Parsanol::Slice, String
+                  slice_or_string_parts << sub_item
                 end
               end
             end
@@ -320,8 +339,8 @@ module Parsanol
               result << item
             when Array
               result.concat(item)
-            when String
-              # Skip unnamed strings when we have named captures
+            when ::Parsanol::Slice, String
+              # Skip unnamed Slices/strings when we have named captures
             end
           end
           return result.length == 1 ? result.first : result
@@ -382,12 +401,22 @@ module Parsanol
 
         # PARSLET SEQUENCE SEMANTICS:
         # If there are named captures (hashes) mixed with other things,
-        # return ONLY the merged hash (discard unnamed strings)
+        # return ONLY the merged hash (discard unnamed Slices/strings)
         return merged_hash unless merged_hash.empty?
 
-        # No named captures - handle strings and other items
-        if string_parts.any?
-          return string_parts.length == 1 ? string_parts.first : string_parts.join
+        # No named captures - handle Slices/strings and other items
+        if slice_or_string_parts.any?
+          # Join Slices/strings, preserving position from first Slice
+          first_slice = slice_or_string_parts.find { |i| i.is_a?(::Parsanol::Slice) }
+          content = slice_or_string_parts.map { |i| i.is_a?(::Parsanol::Slice) ? i.content : i }.join
+
+          if first_slice
+            # Create new Slice with combined content, preserving position from first
+            ::Parsanol::Slice.new(first_slice.offset, content, first_slice.position_cache)
+          else
+            # All plain strings (shouldn't happen with new decode_flat, but handle it)
+            slice_or_string_parts.length == 1 ? slice_or_string_parts.first : content
+          end
         end
 
         # Only other items (arrays, etc.)
@@ -398,36 +427,50 @@ module Parsanol
 
       # Parslet/Parsanol repetition semantics:
       # 1. Return [] for empty repetitions
-      # 2. If all items are strings, join them
+      # 2. If all items are Slices (or strings), join them preserving position
       # 3. Otherwise return array
       def self.flatten_repetition(items)
         return EMPTY_ARRAY if items.empty?
 
         # Single-pass flatten and check
         flat_items = []
-        all_strings = true
+        all_slices_or_strings = true
 
         items.each do |item|
           if item.is_a?(Array)
             item.each do |sub|
               flat_items << sub
-              all_strings = false unless sub.is_a?(String)
+              all_slices_or_strings = false unless slice_or_string?(sub)
             end
           else
             flat_items << item
-            all_strings = false unless item.is_a?(String)
+            all_slices_or_strings = false unless slice_or_string?(item)
           end
         end
 
         return EMPTY_ARRAY if flat_items.empty?
 
-        # If all strings, join them (string-like repetition)
-        if all_strings && flat_items.all?(String)
-          flat_items.join
+        # If all Slices or strings, join them preserving position from first Slice
+        if all_slices_or_strings
+          first_slice = flat_items.find { |i| i.is_a?(::Parsanol::Slice) }
+          content = flat_items.map { |i| i.is_a?(::Parsanol::Slice) ? i.content : i }.join
+
+          if first_slice
+            # Create new Slice with combined content, preserving position from first
+            ::Parsanol::Slice.new(first_slice.offset, content, first_slice.position_cache)
+          else
+            # All plain strings (shouldn't happen with new decode_flat, but handle it)
+            content
+          end
         else
           flat_items
         end
       end
+
+      # Check if value is a Slice or String
+      def self.slice_or_string?(value)
+        value.is_a?(::Parsanol::Slice) || value.is_a?(String)
+      end
     end
 
     private_constant :AstTransformer

data/lib/parsanol/options/zero_copy.rb

@@ -98,7 +98,7 @@ module Parsanol
         raise LoadError,
               "ZeroCopy mode requires native extension for direct FFI object construction. " \
               "Run `rake compile` to build the extension, or use " \
-              "Parsanol::RubyTransform for Ruby-only parsing."
+              "parser.parse(input, mode: :ruby) for pure Ruby parsing."
       end
 
       grammar_json = Parsanol::Native.serialize_grammar(root)

data/lib/parsanol/options.rb

@@ -2,19 +2,20 @@
 
 # Parsanol Transform Mode Options
 #
-# This module provides three transformation modes for parsing:
+# This module provides the ZeroCopy transformation mode for maximum performance:
 #
-# 1. RubyTransform - Parse in Rust/Ruby, Transform in Ruby (default, most flexible)
-# 2. Serialized - Parse + Transform in Rust, JSON output (requires native extension)
-# 3. ZeroCopy - Direct FFI object construction (requires native extension, fastest)
+# ZeroCopy - Direct FFI object construction (requires native extension, fastest)
 #
 # Usage:
 #   class MyParser < Parsanol::Parser
-#     include Parsanol::RubyTransform  # or Serialized, or ZeroCopy
+#     include Parsanol::ZeroCopy
 #     rule(:number) { match('[0-9]').repeat(1).as(:int) }
 #     root(:number)
+#
+#     output_types(number: MyNumberClass)
 #   end
+#
+# For standard parsing, use the Parse Modes API instead:
+#   parser.parse(input, mode: :native)  # or :ruby, :json
 
-require 'parsanol/options/ruby_transform'
-require 'parsanol/options/serialized'
 require 'parsanol/options/zero_copy'

data/lib/parsanol/parser.rb

@@ -82,22 +82,32 @@ module Parsanol
 
     # Unified parsing interface with mode selection support.
     #
+    # All parse modes return results with Slice objects that contain
+    # position information (offset, length, line, column). This enables
+    # source code extraction, error reporting, and remark attachment.
+    #
     # @param input [String] the string to parse
     # @param mode_or_opts [Symbol, Hash] parsing mode or options hash
     # @param kwargs [Hash] additional keyword options
     #
     # Modes:
-    # - :ruby - Pure Ruby parsing (default, always available)
+    # - :ruby - Pure Ruby parsing (always available, returns Slices with position)
     # - :native - Use Rust extension if available, fallback to Ruby
-    # - :json - Return JSON string representation of parse tree
+    # - :json - Return JSON string with position info for each value
     #
     # Options:
     # - :reporter - Custom error reporter instance
     # - :prefix - Allow partial matching (default: false)
     #
-    # @return [Hash, Array, String, Parsanol::Slice] parsed result
+    # @return [Hash, Array, Parsanol::Slice] parsed result with position info
     # @raise [Parsanol::ParseFailed] when parsing fails
     #
+    # @example Parse and access position info
+    #   result = parser.parse("hello")
+    #   result[:name].offset         # => 0
+    #   result[:name].line_and_column # => [1, 1]
+    #   result[:name].to_s           # => "hello"
+    #
     def parse(input, mode_or_opts = {}, **kwargs)
       if mode_or_opts.is_a?(Hash)
         # Legacy API: parse(input, options={})
@@ -150,24 +160,31 @@ module Parsanol
     #
 
     # Native extension parsing with Ruby fallback.
+    # Returns results with Slice objects containing position info.
     #
     # @param input [String] input to parse
     # @param opts [Hash] parsing options
-    # @return [Object] parse result
+    # @return [Object] parse result with Slice objects for position info
     #
     def parse_native(input, opts)
       if Parsanol::Native.available?
-        Parsanol::Native.parse_parslet_compatible(root, input)
+        # Build line cache for position info
+        line_cache = Parsanol::Source::LineCache.new
+        line_cache.scan_for_line_endings(0, input)
+
+        # Parse with position info (now the default)
+        Parsanol::Native.parse_parslet_compatible(root, input, line_cache)
       else
         parse_ruby(input, opts)
       end
     end
 
-    # JSON output mode - returns JSON string representation.
+    # JSON output mode - returns JSON with position info.
+    # All Slice values are serialized with their position information.
     #
     # @param input [String] input to parse
     # @param opts [Hash] parsing options
-    # @return [String] JSON representation of parse tree
+    # @return [String] JSON representation with position info
     #
     def parse_json(input, opts)
       if Parsanol::Native.available?

data/lib/parsanol/slice.rb

@@ -142,5 +142,66 @@ module Parsanol
     def inspect
       "#{content.inspect}@#{offset}"
     end
+
+    # JSON serialization --------------------------------------------------------
+
+    # JSON serialization returns the full object with position info.
+    # This is the default behavior - position info is ALWAYS included.
+    #
+    # @return [String] JSON representation with value and position
+    def to_json(*)
+      as_json.to_json(*)
+    end
+
+    # Returns a hash with full position information for JSON serialization.
+    # Line and column are always included when a position cache is available.
+    #
+    # @return [Hash] hash with value, offset, length, and line/column
+    def as_json(_options = {})
+      result = {
+        'value' => content,
+        'offset' => offset,
+        'length' => length
+      }
+
+      if position_cache
+        line, column = line_and_column
+        result['line'] = line
+        result['column'] = column
+      end
+
+      result
+    end
+
+    # Returns a SourceSpan representing this slice's position
+    #
+    # @param input [String, nil] the original input (needed for line/column)
+    # @return [Parsanol::SourceSpan] span object
+    def to_span(_input = nil)
+      start_pos = if position_cache
+                    line, column = line_and_column
+                    SourcePosition.new(offset: offset, line: line, column: column)
+                  else
+                    SourcePosition.new(offset: offset, line: 1, column: 1)
+                  end
+
+      end_offset = offset + length
+      end_pos = if position_cache
+                  line, column = position_cache.line_and_column(end_offset)
+                  SourcePosition.new(offset: end_offset, line: line, column: column)
+                else
+                  SourcePosition.new(offset: end_offset, line: 1, column: 1)
+                end
+
+      SourceSpan.new(start_pos: start_pos, end_pos: end_pos)
+    end
+
+    # Extract this slice's content from the original input string
+    #
+    # @param input [String] the original input string
+    # @return [String] the slice content extracted from input
+    def extract_from(input)
+      input.byteslice(offset, length)
+    end
   end
 end

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = '1.0.1'
+  VERSION = '1.0.2'
 end

data/{parsanol-ruby.gemspec → parsanol.gemspec}

@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
     LICENSE
     Rakefile
     README.adoc
-    parsanol-ruby.gemspec
+    parsanol.gemspec
     Cargo.toml
     Cargo.lock
   ]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: aarch64-linux
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2026-03-03 00:00:00.000000000 Z
+date: 2026-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -196,7 +196,6 @@ files:
 - lib/parsanol/incremental_parser.rb
 - lib/parsanol/interval_tree.rb
 - lib/parsanol/lazy_result.rb
-- lib/parsanol/lexer.rb
 - lib/parsanol/mermaid.rb
 - lib/parsanol/native.rb
 - lib/parsanol/native/parser.rb
@@ -210,8 +209,6 @@ files:
 - lib/parsanol/optimizers/quantifier_optimizer.rb
 - lib/parsanol/optimizers/sequence_optimizer.rb
 - lib/parsanol/options.rb
-- lib/parsanol/options/ruby_transform.rb
-- lib/parsanol/options/serialized.rb
 - lib/parsanol/options/zero_copy.rb
 - lib/parsanol/parallel.rb
 - lib/parsanol/parser.rb
@@ -244,7 +241,7 @@ files:
 - lib/parsanol/wasm/parsanol.js
 - lib/parsanol/wasm/parslet.d.ts
 - lib/parsanol/wasm_parser.rb
-- parsanol-ruby.gemspec
+- parsanol.gemspec
 homepage: https://github.com/parsanol/parsanol-ruby
 licenses:
 - MIT

data/lib/parsanol/lexer.rb

@@ -1,144 +0,0 @@
-# frozen_string_literal: true
-
-require 'parsanol/native'
-
-module Parsanol
-  # Generic lexer for fast tokenization
-  #
-  # Create a lexer by subclassing and defining tokens:
-  #
-  #   class JsonLexer < Parsanol::Lexer
-  #     token :string, /"[^"]*"/
-  #     token :number, /-?[0-9]+(\.[0-9]+)?/
-  #     token :true, /true/
-  #     token :false, /false/
-  #     token :null, /null/
-  #     token :lbrace, /\{/
-  #     token :rbrace, /\}/
-  #     token :lbracket, /\[/
-  #     token :rbracket, /\]/
-  #     token :colon, /:/
-  #     token :comma, /,/
-  #
-  #     ignore /\s+/
-  #   end
-  #
-  #   lexer = JsonLexer.new
-  #   tokens = lexer.tokenize('{"name": "test"}')
-  #
-  class Lexer
-    class << self
-      # Define a token pattern
-      #
-      # @param name [Symbol] Token type name
-      # @param pattern [Regexp] Pattern to match
-      # @param priority [Integer] Priority for conflict resolution (higher = preferred)
-      # @param block [Proc] Optional block to transform the matched value
-      def token(name, pattern, priority: 0, &block)
-        token_definitions << Definition.new(
-          name: name.to_s,
-          pattern: pattern.source,
-          priority: priority,
-          ignore: false,
-          transform: block
-        )
-      end
-
-      # Define patterns to ignore (e.g., whitespace, comments)
-      #
-      # @param pattern [Regexp] Pattern to ignore
-      def ignore(pattern)
-        token_definitions << Definition.new(
-          name: '__ignore__',
-          pattern: pattern.source,
-          priority: 0,
-          ignore: true,
-          transform: nil
-        )
-      end
-
-      # Define keywords (identifiers with higher priority)
-      #
-      # @param keywords [Array<Symbol>] Keyword names
-      # @param priority [Integer] Priority (default: 100)
-      def keyword(*keywords, priority: 100)
-        keywords.each do |kw|
-          token_definitions << Definition.new(
-            name: kw.to_s.upcase,
-            pattern: Regexp.new(Regexp.escape(kw.to_s), Regexp::IGNORECASE).source,
-            priority: priority,
-            ignore: false,
-            transform: nil
-          )
-        end
-      end
-
-      # Get token definitions for this lexer class
-      #
-      # @return [Array<Definition>] Token definitions
-      def token_definitions
-        @token_definitions ||= []
-      end
-
-      # Inherit token definitions from parent class
-      def inherited(subclass)
-        super
-        subclass.instance_variable_set(:@token_definitions, token_definitions.dup)
-      end
-    end
-
-    # Token definition
-    Definition = Struct.new(:name, :pattern, :priority, :ignore, :transform)
-
-    # Initialize the lexer
-    def initialize
-      @lexer_id = nil
-      @transforms = build_transforms
-    end
-
-    # Tokenize input string
-    #
-    # @param input [String] Input to tokenize
-    # @return [Array<Hash>] Array of tokens with type, value, and location
-    def tokenize(input)
-      ensure_lexer_created
-
-      tokens = Native.tokenize_with_lexer(@lexer_id, input)
-
-      # Apply any transforms
-      tokens.map do |token|
-        transform = @transforms[token['type']]
-        if transform
-          token = token.dup
-          token['value'] = transform.call(token['value'])
-        end
-        token
-      end
-    end
-
-    private
-
-    def ensure_lexer_created
-      return if @lexer_id
-
-      definitions = self.class.token_definitions.map do |d|
-        {
-          'name' => d.name,
-          'pattern' => d.pattern,
-          'priority' => d.priority,
-          'ignore' => d.ignore
-        }
-      end
-
-      @lexer_id = Native.create_lexer(definitions)
-    end
-
-    def build_transforms
-      transforms = {}
-      self.class.token_definitions.each do |d|
-        transforms[d.name] = d.transform if d.transform && !d.ignore
-      end
-      transforms
-    end
-  end
-end