parsanol 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08c90d92486d3df8bc341518c5e16c2314bdeb5189581d977e47cf01994e7af4'
-  data.tar.gz: 8bbc6ef440807d2a51fe3dce9d46963f6b2788aef594ed362d8f20add00423bf
+  metadata.gz: d6a6abf1fc72b2167ed8fb0b494128d8a7e219396954c2a1e81b2360c6ff2c21
+  data.tar.gz: 6465b111339a2ec6dddbc7e63b1116ff7ed0f0e3c02e490bfdd5ba939fb338cc
 SHA512:
-  metadata.gz: 299847d77833bcf41edcd0ca55e0350757fc833d6a9ce86570f927aae8a6a2ebeee102c671d2267ec6bf3462a27754b71fb136e7450502edc925d68eb604ffd8
-  data.tar.gz: c2dcf0b8bc570a5c46fd259eb8e17bc02e62515adf3c30a3aefdb1c21d1c47e7d9f0f75364f3d091386c6663112de79350414db3d6bda49cb34895fda342c0ab
+  metadata.gz: 591469089373168dfa0293b027097054a29b1a2e0353f5ed08224999965af5c2a49ed3fe2041cdee8a206d1527e91b2899ad7a9062f17eeb75ac13d43f928048
+  data.tar.gz: '0871ec48005b0d7df2a79b7d4c269aca9e1953733e45fad55a1c4126d3916389eb2879634246a8c3a2a69f02bd8ecbfeb363ee908dc212a3ce20f369ccc057ff'

data/README.adoc

@@ -55,9 +55,8 @@ gem install parsanol
 
 == Usage
 
+[[basic-parsing]]
 === Basic Parser
-<<<basic-parsing>>
-
 Define parsers by creating a class that inherits from `Parsanol::Parser` and declaring rules:
 
 [source,ruby]
@@ -74,9 +73,8 @@ parser = MyParser.new
 result = parser.parse('if(x)')
 ----
 
+[[error-reporting]]
 === Error Reporting
-<<<error-reporting>>
-
 Parsanol provides detailed error messages when parsing fails:
 
 [source,ruby]
@@ -89,9 +87,8 @@ rescue Parsanol::ParseFailed => e
 end
 ----
 
+[[transformation]]
 === Transformation
-<<<transformation>>
-
 Convert parse trees to AST using pattern-based transformations:
 
 [source,ruby]
@@ -104,9 +101,8 @@ end
 ast = MyTransform.new.apply(parse_tree)
 ----
 
+[[native-extension]]
 === Native Extension
-<<<native-extension>>
-
 For maximum performance, compile the Rust native extension:
 
 [source,shell]
@@ -118,21 +114,84 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 bundle exec rake compile
 ----
 
+[[slice-support]]
 === Slice Support
-<<<slice-support>>
 
-Parsanol preserves source positions for each parsed element:
+All parse results include source position information through `Parsanol::Slice` objects:
+
+[source,ruby]
+----
+# Parse returns results with position info
+result = parser.parse("hello world", mode: :native)
+name = result[:name]
+
+# Access the value
+name.to_s     # => "hello"
+
+# Access position information
+name.offset   # => 0 (byte offset in original input)
+name.length   # => 5
+name.line_and_column  # => [1, 1] (1-indexed)
+
+# Compare with strings (Slice compares by content)
+name == "hello"  # => true
+
+# Extract from original source
+name.extract_from(input)  # => "hello"
+----
+
+==== JSON Output with Position Info
+
+When using JSON mode, position information is included inline with each value:
 
 [source,ruby]
 ----
-# Result includes position information
-[{"word" => "hello"@0}, " "@5, {"name" => "world"@6}]
+result = parser.parse("hello", mode: :json)
+# => {
+#   "name": {
+#     "value": "hello",
+#     "offset": 0,
+#     "length": 5,
+#     "line": 1,
+#     "column": 1
+#   }
+# }
+----
+
+This format ensures position information is available for all downstream consumers including IDEs, linters, and error reporting tools.
+
+==== Slice API
 
-# The @N notation shows the byte offset in the original input
-# Parsanol::Slice is fully compatible with Parslet::Slice
+[source,ruby]
+----
+class Parsanol::Slice
+  # Core attributes
+  def content       # String content
+  def offset        # Byte offset in original input
+  def length        # Length of the slice
+  def line_and_column  # [line, column] tuple (requires line cache)
+
+  # String compatibility
+  def to_s          # Returns content
+  def to_str        # Implicit string conversion
+  def ==(other)     # Compares content with String or Slice
+
+  # JSON serialization
+  def to_json       # Returns { "value" => ..., "offset" => ..., ... }
+  def as_json       # Returns hash with position info
+
+  # Utility
+  def to_span(input)  # Returns SourceSpan object
+  def extract_from(input)  # Extracts content from original input
+end
 ----
 
-This is essential for linters, IDEs, and tools that need to map parsed elements back to source locations.
+This is essential for:
+
+* **Linters** - Map errors back to source locations
+* **IDEs** - Provide go-to-definition, hover info
+* **Comment attachment** - Attach remarks to AST nodes by position
+* **Source extraction** - Get original text for any parsed element
 
 == Migrating from Parslet
 
@@ -222,23 +281,196 @@ parser.parse('123')  # Works exactly the same
                         └─────────────────────────────────────┘
 ----
 
-=== Performance Modes
+=== Parse Modes
+
+Parsanol offers 3 parsing modes through the `parse` method. All modes return `Parsanol::Slice` objects with position information:
 
-Parsanol offers multiple parsing modes with different performance characteristics:
+[source,ruby]
+----
+result = parser.parse(input, mode: :native)  # mode is optional, :native is default
+----
 
-[cols="4,2,2,3"]
+[cols="1,1,1,2,2"]
 |===
-| Mode | Speed | Use Case | How It Works
+| Mode | Backend | Keys | Values | Best For
 
-| Pure Ruby | 1x (baseline) | Compatibility, debugging | Ruby parsing engine
-| Native Batch | ~20x | Need Ruby objects | Rust parsing, AST via u64
-| Native ZeroCopy | ~25x | Maximum performance | Direct FFI construction
-| Native ZeroCopy + Slice | ~29x | Linters, IDEs | Zero-copy with positions
+| `:ruby` | Pure Ruby | Symbol | Slice | Debugging, fallback
+| `:native` | Rust FFI | Symbol | Slice | **Production (DEFAULT)**
+| `:json` | Rust FFI | String | Hash + position | APIs, serialization
 |===
 
-== Streaming Builder API
-<<<streaming-builder>>
+All modes include position info (offset, length, line, column) by default.
+
+==== Mode Details
+
+**Ruby Mode** (`:ruby`)::
+Pure Ruby parsing engine. Use for debugging grammar issues or when native extension is unavailable.
+
+**Native Mode** (`:native`)::
+Rust parser via FFI with automatic transformation to Ruby-friendly format (Symbol keys). ~20x faster than pure Ruby. This is the **default mode**.
+
+**JSON Mode** (`:json`)::
+Rust parser that returns JSON-serializable output with inline position information. Use for APIs and when you need JSON-compatible output.
+
+=== ZeroCopy Interface (Low-Level API)
+
+For maximum performance (~29x faster than pure Ruby), use the ZeroCopy interface which bypasses Ruby transformation:
+
+[source,ruby]
+----
+# Low-level API: Direct Rust access, String keys
+grammar = Parsanol::Native.serialize_grammar(parser.root)
+result = Parsanol::Native.parse_to_ruby_objects(grammar, input)
+# Returns: { "name" => Slice("hello", offset: 0, length: 5) }
+
+# High-level ZeroCopy: Include module for direct Ruby objects
+class FastParser < Parsanol::Parser
+  include Parsanol::ZeroCopy
+
+  rule(:number) { match('[0-9]').repeat(1) }
+  root(:number)
+
+  output_types(number: MyNumberClass)  # Map to Ruby classes
+end
+
+parser = FastParser.new
+expr = parser.parse("42")  # Returns MyNumberClass instance directly
+----
+
+[cols="1,2,2"]
+|===
+| Method | Keys | Use Case
+
+| `parse_to_ruby_objects` | String | Low-level, Slice objects directly from Rust
+| `Parsanol::ZeroCopy` module | Ruby objects | Maximum performance, direct object construction
+|===
+
+NOTE: ZeroCopy requires the native extension and type mapping definitions.
+
+==== When to Use Parse Modes vs ZeroCopy
+
+[cols="1,2,2"]
+|===
+| Your Need | Use This | Why
+
+| Building an API | JSON mode (`:json`) | Direct JSON serialization
+| Building a linter/IDE | Native mode (`:native`) | Position info for errors
+| Need position info | Parse Modes (not ZeroCopy) | ZeroCopy skips position tracking
+| High-throughput parsing | ZeroCopy | Maximum performance
+| Type-safe AST with methods | ZeroCopy | Direct typed object construction
+| Debugging grammar | Ruby mode (`:ruby`) | Pure Ruby, easier to trace
+|===
+
+==== ZeroCopy Example: Calculator with Direct Object Construction
+
+[source,ruby]
+----
+# 1. Define your AST classes with methods
+module Calculator
+  class Expr
+    def eval = raise NotImplementedError
+  end
+
+  class Number < Expr
+    attr_reader :value
+    def initialize(value) = @value = value
+    def eval = @value
+  end
+
+  class BinOp < Expr
+    attr_reader :left, :op, :right
+    def initialize(left:, op:, right:)
+      @left, @op, @right = left, op, right
+    end
+    def eval
+      case @op
+      when '+' then @left.eval + @right.eval
+      when '-' then @left.eval - @right.eval
+      when '*' then @left.eval * @right.eval
+      when '/' then @left.eval / @right.eval
+      end
+    end
+  end
+end
+
+# 2. Define parser with ZeroCopy and output_types
+class CalculatorParser < Parsanol::Parser
+  include Parsanol::ZeroCopy
+
+  rule(:number) { match('[0-9]').repeat(1).as(:int) }
+  rule(:expression) { (number.as(:left) >> add_op >> expression.as(:right)).as(:binop) | number }
+  root(:expression)
+
+  # Map rules to Ruby classes - Rust constructs these directly!
+  output_types(
+    number: Calculator::Number,
+    binop: Calculator::BinOp
+  )
+end
+
+# 3. Parse and evaluate - no transform needed!
+parser = CalculatorParser.new
+expr = parser.parse("2 + 3 * 4")  # Returns Calculator::BinOp directly
+puts expr.eval  # => 14 (with proper precedence)
+----
+
+==== Low-Level ZeroCopy: `parse_to_ruby_objects`
+
+When you don't need typed objects, use `parse_to_ruby_objects` for direct Slice access:
+
+[source,ruby]
+----
+# Direct FFI call - bypasses transformation, String keys
+grammar = Parsanol::Native.serialize_grammar(MyParser.new.root)
+result = Parsanol::Native.parse_to_ruby_objects(grammar, input)
 
+# Result structure (String keys, Slice values):
+# { "name" => Slice("hello", offset: 0, length: 5),
+#   "value" => Slice("42", offset: 10, length: 2) }
+
+# Access position info directly
+result["name"].offset    # => 0
+result["name"].to_s      # => "hello"
+----
+
+==== ZeroCopy Requirements
+
+The ZeroCopy module requires:
+
+1. **Native extension** - Run `bundle exec rake compile`
+2. **Type mapping** - Define `output_types` in your parser
+3. **Matching constructors** - Your Ruby classes must accept the parsed attributes
+
+For complex types, you may also need Rust-side type definitions with `#[derive(RubyObject)]` for full zero-copy FFI construction.
+
+=== Parsing Backends (Rust Core)
+
+Behind the scenes, the Rust implementation uses one of two parsing backends:
+
+[cols="2,2,3"]
+|===
+| Backend | Use Case | Characteristics
+
+| Packrat (default) | Complex grammars | O(n) guaranteed, higher memory
+| Bytecode VM | Simple patterns | Lower memory, faster for linear patterns
+| Auto | Variable workloads | Analyzes grammar, selects best backend
+|===
+
+The Ruby bindings automatically use the best backend for your grammar:
+
+* Uses `Backend::Auto` by default (same as parsanol-rs)
+* Detects nested repetitions, overlapping choices
+* Recommends Packrat for complex grammars
+* Falls back to Bytecode for simple patterns
+
+NOTE: The backend selection is transparent to Ruby users. The parser object automatically uses the optimal backend based on grammar analysis.
+
+For more details on backend selection and grammar analysis, see the https://parsanol.github.io/backends[Parsing Backends documentation].
+
+
+
+[[streaming-builder]]
+== Streaming Builder API
 For maximum performance, use the streaming builder API which eliminates intermediate AST construction:
 
 [source,ruby]
@@ -289,9 +521,8 @@ result = Parsanol::Native.parse_with_builder(grammar, input, builder)
 | `finish` | Parsing complete | Returns nil
 |===
 
+[[parallel-parsing]]
 == Parallel Parsing
-<<<parallel-parsing>>
-
 Parse multiple inputs using all CPU cores:
 
 [source,ruby]
@@ -312,9 +543,8 @@ config = Parsanol::Parallel::Config.new
 results = Parsanol::Parallel.parse_batch(grammar, inputs, config: config)
 ----
 
+[[infix-expressions]]
 == Infix Expression Parsing
-<<<infix-expressions>>
-
 Built-in support for parsing infix expressions with operator precedence:
 
 [source,ruby]
@@ -336,9 +566,8 @@ class CalculatorParser < Parsanol::Parser
 end
 ----
 
+[[treetop-expressions]]
 == Treetop Expression Syntax
-<<<treetop-expressions>>
-
 Parsanol supports treetop-style expression strings for quick grammar definition:
 
 [source,ruby]
@@ -392,9 +621,8 @@ grammar = Parsanol::Native.serialize_grammar(atom)
 Parsanol::Native.parse_to_ruby_objects(grammar, 'aaa')
 ----
 
+[[security-features]]
 == Security Features
-<<<security-features>>
-
 For parsing untrusted input, use built-in limits:
 
 [source,ruby]
@@ -407,9 +635,8 @@ result = Parsanol::Native.parse_with_limits(
 )
 ----
 
+[[debug-tools]]
 == Debug Tools
-<<<debug-tools>>
-
 Enable tracing for debugging grammars:
 
 [source,ruby]

data/Rakefile

@@ -1,7 +1,13 @@
 # frozen_string_literal: true
 
 require 'bundler/gem_tasks'
-require 'rspec/core/rake_task'
+
+begin
+  require 'rspec/core/rake_task'
+rescue LoadError
+  # RSpec not available in this environment
+end
+
 require 'rdoc/task'
 require 'rubygems/package_task'
 
@@ -11,7 +17,7 @@ rescue LoadError, NoMethodError
   # Opal not available or incompatible with current Ruby version
 end
 
-GEMSPEC = Gem::Specification.load('parsanol-ruby.gemspec')
+GEMSPEC = Gem::Specification.load('parsanol.gemspec')
 
 # Load rake tasks from rakelib/
 Dir.glob('rakelib/*.rake').each { |r| load r }
@@ -60,7 +66,7 @@ namespace :gem do
 
   desc 'Define the gem task to build on any platform (compile on install)'
   task 'platform:any' do
-    spec = Gem::Specification.load('parsanol-ruby.gemspec').dup
+    spec = Gem::Specification.load('parsanol.gemspec').dup
     task = Gem::PackageTask.new(spec)
     task.define
   end

data/ext/parsanol_native/Cargo.toml

@@ -2,7 +2,8 @@
 #
 # Rust extension for parsanol-ruby that provides:
 # - Fast parsing with packrat memoization
-# - Three transform modes: RubyTransform, Serialized, ZeroCopy
+# - Three parse modes: ruby, native, json
+# - ZeroCopy interface for maximum performance
 # - Source location tracking
 # - Streaming parsing
 # - Incremental parsing for editor integration
@@ -11,7 +12,7 @@
 
 [package]
 name = "parsanol_native"
-version = "1.0.0"
+version = "1.0.2"
 edition = "2021"
 rust-version = "1.75"
 

data/lib/parsanol/native/parser.rb

@@ -13,6 +13,7 @@ module Parsanol
     #
     module Parser
       # Two-level grammar cache (module-level for proper initialization)
+      # These MUST be mutable for caching to work
       GRAMMAR_HASH_CACHE = Hash.new  # object_id => hash_key
       GRAMMAR_CACHE = Hash.new       # hash_key => grammar_json
 
@@ -35,42 +36,50 @@ module Parsanol
         # Parse using native engine
         # @param grammar_json [String] JSON-serialized grammar
         # @param input [String] Input string to parse
-        # @return Ruby AST from parsing
-        def parse(grammar_json, input)
+        # @param line_cache [Parsanol::Source::LineCache, nil] Optional line cache for position info
+        # @return Ruby AST from parsing with Slice objects for strings
+        def parse(grammar_json, input, line_cache = nil)
           raise LoadError, 'Native parser not available. Run `rake compile` to build.' unless available?
 
+          # Build line cache if not provided
+          line_cache ||= build_line_cache(input)
+
           # Call native parse_batch (returns flat u64 array)
           flat = Parsanol::Native.parse_batch(grammar_json, input)
-          # Decode flat array to Ruby AST
-          decode_flat(flat, input)
+          # Decode flat array to Ruby AST with Slice objects
+          decode_flat(flat, input, line_cache)
         end
 
         # Parse a grammar with automatic serialization and caching
         # @param root_atom [Parsanol::Atoms::Base] Root atom of the grammar
         # @param input [String] Input string to parse
-        # @return Ruby AST from parsing
-        def parse_with_grammar(root_atom, input)
+        # @param line_cache [Parsanol::Source::LineCache, nil] Optional line cache
+        # @return Ruby AST from parsing with Slice objects
+        def parse_with_grammar(root_atom, input, line_cache = nil)
           # Extract root atom if a Parser is passed
           root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
           grammar_json = serialize_grammar(root_atom)
-          parse(grammar_json, input)
+          parse(grammar_json, input, line_cache)
         end
 
         # Parse and transform to Parslet-compatible format
+        # NOTE: This method now returns Slice objects with position info by default.
+        # The name is kept for backward compatibility but it's now the primary parse method.
         # @param root_atom [Parsanol::Atoms::Base] Root atom of the grammar
         # @param input [String] Input string to parse
-        # @return Ruby AST in Parslet-compatible format
-        def parse_parslet_compatible(root_atom, input)
+        # @param line_cache [Parsanol::Source::LineCache, nil] Optional line cache
+        # @return Ruby AST in Parslet-compatible format with Slice objects
+        def parse_parslet_compatible(root_atom, input, line_cache = nil)
           # Extract root atom if a Parser is passed
           root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
-          raw_ast = parse_with_grammar(root_atom, input)
+          raw_ast = parse_with_grammar(root_atom, input, line_cache)
           AstTransformer.transform(raw_ast)
         end
 
         # Parse multiple inputs with the same grammar (more efficient)
         # @param root_atom [Parsanol::Atoms::Base] Root atom of the grammar
         # @param inputs [Array<String>] Array of input strings to parse
-        # @return [Array] Array of raw Ruby ASTs from parsing
+        # @return [Array] Array of Ruby ASTs with Slice objects
         def parse_batch_inputs(root_atom, inputs)
           # Extract root atom if a Parser is passed
           root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
@@ -81,7 +90,7 @@ module Parsanol
         # Parse multiple inputs with transformation
         # @param root_atom [Parsanol::Atoms::Base] Root atom of the grammar
         # @param inputs [Array<String>] Array of input strings to parse
-        # @return [Array] Array of transformed Ruby ASTs
+        # @return [Array] Array of transformed Ruby ASTs with Slice objects
         def parse_batch_with_transform(root_atom, inputs)
           # Extract root atom if a Parser is passed
           root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
@@ -95,13 +104,22 @@ module Parsanol
         # Parse without transformation (faster for raw AST access)
         # @param root_atom [Parsanol::Atoms::Base] Root atom of the grammar
         # @param input [String] Input string to parse
-        # @return Raw Ruby AST from parsing (native format)
+        # @return Raw Ruby AST from parsing with Slice objects
         def parse_raw(root_atom, input)
           # Extract root atom if a Parser is passed
           root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
           parse_with_grammar(root_atom, input)
         end
 
+        # Build a line cache for an input string
+        # @param input [String] The input string
+        # @return [Parsanol::Source::LineCache] The line cache
+        def build_line_cache(input)
+          cache = ::Parsanol::Source::LineCache.new
+          cache.scan_for_line_endings(0, input)
+          cache
+        end
+
         # Serialize a grammar to JSON, with two-level caching
         # Level 1: object_id => hash_key (avoids grammar traversal)
         # Level 2: hash_key => grammar_json (avoids serialization)
@@ -471,13 +489,19 @@ module Parsanol
         #   0x01 = bool
         #   0x02 = int
         #   0x03 = float
-        #   0x04 = string_ref (offset, length)
+        #   0x04 = string_ref (offset, length) - creates Slice with position info
         #   0x05 = array_start
         #   0x06 = array_end
         #   0x07 = hash_start
         #   0x08 = hash_end
         #   0x09 = hash_key (tag, len, key_chunks..., value)
-        def decode_flat(flat, input)
+        #   0x0A = inline_string (interned string from arena)
+        #
+        # @param flat [Array<Integer>] Flat u64 array from native parser
+        # @param input [String] Original input string
+        # @param line_cache [Parsanol::Source::LineCache, nil] Line cache for position info
+        # @return Ruby AST with Slice objects for all string values
+        def decode_flat(flat, input, line_cache = nil)
           stack = []
           i = 0
 
@@ -500,10 +524,12 @@ module Parsanol
               float = [bits].pack('Q').unpack1('D')
               stack << float
               i += 2
-            when 0x04 # string_ref (from input)
+            when 0x04 # string_ref (from input) - create Slice with position info
               offset = flat[i + 1]
               length = flat[i + 2]
-              stack << input.byteslice(offset, length)
+              content = input.byteslice(offset, length)
+              # Create Slice with position info - this is the key change
+              stack << ::Parsanol::Slice.new(offset, content, line_cache)
               i += 3
             when 0x0A # inline_string (interned string from arena)
               # Format: tag, len, u64 chunks of string bytes
@@ -523,7 +549,9 @@ module Parsanol
               end
               i += chunks
 
-              stack << bytes.pack('C*').force_encoding('UTF-8')
+              # Inline strings don't have source position, use Slice with offset 0
+              content = bytes.pack('C*').force_encoding('UTF-8')
+              stack << ::Parsanol::Slice.new(0, content, nil)
             when 0x05 # array_start
               stack << :array_marker
               i += 1