parsanol 3.0.0

data/example/calculator/transform.rb

@@ -0,0 +1,153 @@
+# A simple integer calculator to answer the question about how to do 
+# left and right associativity in parslet (PEG) once and for all. 
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'rspec'
+require 'parsanol/parslet'
+require 'parsanol/rig/rspec'
+
+# This is the parsing stage. It expresses left associativity by compiling
+# list of things that have the same associativity. 
+class CalcParser < Parsanol::Parser
+  root :addition
+  
+  rule(:addition) {
+    multiplication.as(:l) >> (add_op >> multiplication.as(:r)).repeat(1) |
+    multiplication
+  }
+  
+  rule(:multiplication) { 
+    integer.as(:l) >> (mult_op >> integer.as(:r)).repeat(1) |
+    integer }
+  
+  rule(:integer) { digit.repeat(1).as(:i) >> space? }
+
+  rule(:mult_op) { match['*/'].as(:o) >> space? }
+  rule(:add_op) { match['+-'].as(:o) >> space? }
+
+  rule(:digit) { match['0-9'] }
+  rule(:space?) { match['\s'].repeat }
+end
+
+# Classes for the abstract syntax tree.
+Int    = Struct.new(:int) {
+  def eval; self end
+  def op(operation, other)
+    left = int
+    right = other.int 
+
+    Int.new(
+      case operation
+        when '+'
+          left + right
+        when '-'
+          left - right
+        when '*'
+          left * right
+        when '/'
+          left / right
+      end)
+  end
+  def to_i
+    int
+  end
+}
+Seq    = Struct.new(:sequence) {
+  def eval
+    sequence.reduce { |accum, operation| 
+      operation.call(accum) }
+  end
+}
+LeftOp = Struct.new(:operation, :right) {
+  def call(left)
+    left = left.eval
+    right = self.right.eval
+
+    left.op(operation, right)
+  end
+}
+
+# Transforming intermediary syntax tree into a real AST.
+class CalcTransform < Parsanol::Transform
+  rule(i: simple(:i)) { Int.new(Integer(i)) }
+  rule(o: simple(:o), r: simple(:i)) { LeftOp.new(o, i) }
+  rule(l: simple(:i)) { i }
+  rule(sequence(:seq)) { Seq.new(seq) }
+end
+
+# And this calls everything in the right order.
+def calculate(str)
+  intermediary_tree = CalcParser.new.parse(str)
+  abstract_tree = CalcTransform.new.apply(intermediary_tree)
+  result = abstract_tree.eval
+  
+  result.to_i
+end
+
+# A test suite for the above parser
+describe CalcParser do
+  let(:p) { described_class.new }
+  describe '#integer' do
+    let(:i) { p.integer }
+    it "parses integers" do
+      i.should parse('1')
+      i.should parse('123')
+    end 
+    it "consumes trailing white space" do
+      i.should parse('123   ')
+    end 
+    it "doesn't parse floats" do
+      i.should_not parse('1.3')
+    end 
+  end
+  describe '#multiplication' do
+    let(:m) { p.multiplication }
+    it "parses simple multiplication" do
+      m.should parse('1*2')
+    end
+    it "parses division" do
+      m.should parse('1/2')
+    end 
+  end
+  describe '#addition' do
+    let(:a) { p.addition }
+    
+    it "parses simple addition" do
+      a.should parse('1+2')
+      a.should parse('1+2+3-4')
+    end 
+  end
+end
+describe CalcTransform do
+  def t(obj)
+    described_class.new.apply(obj)
+  end
+  
+  it "transforms integers" do
+    t(i: '1').should == Int.new(1)
+  end 
+  it "unwraps left operand" do
+    t(l: :obj).should == :obj
+  end 
+end
+describe 'whole computation specs' do
+  def self.result_of(str, int)
+    it(str) { calculate(str).should == int } 
+  end
+
+  result_of '1+1', 2
+  result_of '1-1-1', -1
+  result_of '1+1+3*5/2', 9
+  result_of '123*2', 246
+end
+
+
+# Enable these if you want to change the code.
+# RSpec::Core::Runner.run([], $stderr, $stdout)
+
+str = ARGV.join
+str = '123*2' if str.match(/^\s*$/)
+
+print "#{str} (command line): -> "
+puts calculate(str)

data/example/calculator/zero_copy.rb

@@ -0,0 +1,269 @@
+# Calculator Example - ZeroCopy: Mirrored Objects (Direct FFI)
+#
+# This example demonstrates ZeroCopy where:
+# 1. Rust parser (parsanol-rs) does the parsing
+# 2. Rust transform converts to typed structs
+# 3. Direct Ruby object construction via FFI (no serialization!)
+# 4. Maximum performance with zero-copy
+#
+# This option provides the best performance but requires type definitions
+# in both Rust and Ruby.
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'parsanol'
+
+# NOTE: This example requires:
+# 1. Native extension support for parse_to_objects
+# 2. #[derive(RubyObject)] proc macro in Rust
+# 3. Matching Ruby class definitions
+#
+# This serves as an API preview.
+
+# Step 1: Define Ruby classes that mirror Rust struct definitions
+# These classes MUST match the Rust definitions exactly
+module Calculator
+  class Expr
+    def eval
+      raise NotImplementedError
+    end
+  end
+
+  class Number < Expr
+    attr_reader :value
+
+    # This constructor is called directly from Rust FFI
+    def initialize(value)
+      @value = value
+    end
+
+    def eval = @value
+
+    def to_s = @value.to_s
+
+    def ==(other)
+      other.is_a?(Number) && @value == other.value
+    end
+  end
+
+  class BinOp < Expr
+    attr_reader :left, :op, :right
+
+    # This constructor is called directly from Rust FFI
+    # Rust sets instance variables directly via rb_ivar_set
+    def initialize(left: nil, op: nil, right: nil)
+      @left = left
+      @op = op
+      @right = right
+    end
+
+    def eval
+      left_val = @left.eval
+      right_val = @right.eval
+
+      case @op
+      when '+' then left_val + right_val
+      when '-' then left_val - right_val
+      when '*' then left_val * right_val
+      when '/' then left_val / right_val
+      end
+    end
+
+    def to_s
+      "(#{@left} #{@op} #{@right})"
+    end
+  end
+end
+
+# Step 2: Define the parser with output type mapping
+class CalculatorParser < Parsanol::Parser
+  # Include ZeroCopy module for direct FFI object construction
+  # include Parsanol::ZeroCopy
+
+  root :expression
+
+  rule(:expression) {
+    (term.as(:left) >> add_op.as(:op) >> expression.as(:right)).as(:binop) |
+    term
+  }
+
+  rule(:term) {
+    (factor.as(:left) >> mult_op.as(:op) >> term.as(:right)).as(:binop) |
+    factor
+  }
+
+  rule(:factor) {
+    lparen >> expression >> rparen |
+    number
+  }
+
+  rule(:number) {
+    (match('[0-9]').repeat(1)).as(:int) >> space?
+  }
+
+  rule(:add_op) { match('[+-]').as(:op) >> space? }
+  rule(:mult_op) { match('[*/]').as(:op) >> space? }
+
+  rule(:lparen) { str('(') >> space? }
+  rule(:rparen) { str(')') >> space? }
+  rule(:space?) { match('\s').repeat }
+
+  # Output type mapping (ZeroCopy feature)
+  # This tells Rust which Ruby classes to construct
+  # output_types(
+  #   number: Calculator::Number,
+  #   binop: Calculator::BinOp
+  # )
+end
+
+# Step 3: Parse with direct object construction
+def calculate(input)
+  parser = CalculatorParser.new
+
+  # ZeroCopy: Parse and get direct Ruby objects
+  # NOTE: This requires native extension support
+  # expr = parser.parse(input)
+  # # expr is already a Calculator::Number or Calculator::BinOp!
+  # # No transform needed, no JSON serialization!
+
+  # For demonstration, simulate what ZeroCopy would return
+  # Real implementation would call:
+  #   Native.parse_to_objects(grammar_json, input, output_types)
+
+  # Simulate direct object construction
+  expr = simulate_parse(input)
+  puts "AST: #{expr.class} -> #{expr.to_s}"
+
+  result = expr.eval
+  puts "Result: #{result}"
+
+  result
+end
+
+# Simulated parsing for demonstration
+def simulate_parse(input)
+  case input.strip
+  when "42"
+    Calculator::Number.new(42)
+  when "1 + 2"
+    Calculator::BinOp.new(
+      left: Calculator::Number.new(1),
+      op: '+',
+      right: Calculator::Number.new(2)
+    )
+  when "3 * 4"
+    Calculator::BinOp.new(
+      left: Calculator::Number.new(3),
+      op: '*',
+      right: Calculator::Number.new(4)
+    )
+  when "2 + 3 * 4"
+    Calculator::BinOp.new(
+      left: Calculator::Number.new(2),
+      op: '+',
+      right: Calculator::BinOp.new(
+        left: Calculator::Number.new(3),
+        op: '*',
+        right: Calculator::Number.new(4)
+      )
+    )
+  when "(2 + 3) * 4"
+    Calculator::BinOp.new(
+      left: Calculator::BinOp.new(
+        left: Calculator::Number.new(2),
+        op: '+',
+        right: Calculator::Number.new(3)
+      ),
+      op: '*',
+      right: Calculator::Number.new(4)
+    )
+  else
+    raise "Not simulated: #{input}"
+  end
+end
+
+# Example usage
+if __FILE__ == $0
+  puts "=" * 60
+  puts "Calculator Example - ZeroCopy: Mirrored Objects"
+  puts "=" * 60
+  puts
+  puts "NOTE: This example shows the planned API for ZeroCopy."
+  puts "The native extension support for parse_to_objects is coming soon."
+  puts
+
+  test_cases = [
+    ["42", 42],
+    ["1 + 2", 3],
+    ["3 * 4", 12],
+    ["2 + 3 * 4", 14],
+    ["(2 + 3) * 4", 20],
+  ]
+
+  test_cases.each do |input, expected|
+    puts
+    puts "-" * 40
+    puts "Input: #{input}"
+    begin
+      result = calculate(input)
+      status = result == expected ? "✓ PASS" : "✗ FAIL"
+      puts "Expected: #{expected}, Got: #{result} - #{status}"
+    rescue => e
+      puts "Error: #{e.message}"
+      puts "✗ FAIL"
+    end
+  end
+
+  puts
+  puts "=" * 60
+  puts "ZeroCopy Benefits:"
+  puts "- FASTEST: No serialization overhead"
+  puts "- Zero-copy: Direct Ruby object construction"
+  puts "- Type-safe: Types defined in both Rust and Ruby"
+  puts "- Methods defined in Ruby (eval, to_s, etc.)"
+  puts
+  puts "ZeroCopy Requirements:"
+  puts "- Define types in Rust with #[derive(RubyObject)]"
+  puts "- Define matching Ruby classes"
+  puts "- Native extension compiled with ruby feature"
+  puts "=" * 60
+end
+
+# Rust code that would be needed (for reference):
+#
+# // In parsanol-rs
+# use parsanol_ruby_derive::RubyObject;
+#
+# #[derive(Debug, Clone, RubyObject)]
+# #[ruby_class("Calculator::Expr")]
+# pub enum Expr {
+#     #[ruby_variant("number")]
+#     Number(i64),
+#
+#     #[ruby_variant("binop")]
+#     BinOp {
+#         left: Box<Expr>,
+#         op: String,
+#         right: Box<Expr>,
+#     },
+# }
+#
+# // The proc macro generates:
+# impl RubyObject for Expr {
+#     fn to_ruby(&self, ruby: &Ruby) -> Result<Value, Error> {
+#         match self {
+#             Expr::Number(n) => {
+#                 let class = ruby.class("Calculator::Number")?;
+#                 class.new_instance((*n,))
+#             }
+#             Expr::BinOp { left, op, right } => {
+#                 let class = ruby.class("Calculator::BinOp")?;
+#                 let obj = class.new_instance()?;
+#                 obj.ivar_set("@left", left.to_ruby(ruby)?)?;
+#                 obj.ivar_set("@op", op.to_ruby(ruby)?)?;
+#                 obj.ivar_set("@right", right.to_ruby(ruby)?)?;
+#                 Ok(obj.as_value())
+#             }
+#         }
+#     }
+# }

data/example/calculator/zero_copy.rb.md

@@ -0,0 +1,36 @@
+# Calculator (Zero-Copy - Option C)
+
+## Purpose
+
+This implementation demonstrates direct FFI object construction: Rust parses
+and directly constructs Ruby objects without serialization.
+
+## When to Use
+
+- Maximum performance required
+- Production systems
+- When zero-copy is critical
+
+## Key Concepts
+
+1. **Direct FFI**: No serialization overhead
+2. **Ruby Object Construction**: Direct via rb_funcall
+3. **Type Safety**: Mirrored types on both sides
+
+## Running
+
+```bash
+ruby example/calculator/zero_copy.rb
+```
+
+## Output
+
+```
+Input: 42+8
+Result: Calculator::AddExpr
+Value: 50
+```
+
+## Note
+
+This is the fastest option but requires more complex FFI setup.

data/example/capture/basic.rb

@@ -0,0 +1,49 @@
+
+# This example demonstrates how pieces of input can be captured and matched
+# against later on. Without this, you cannot match here-documents and other
+# self-dependent grammars. 
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require 'parsanol/parslet'
+require 'parsanol/convenience'
+require 'pp'
+
+class CapturingParser < Parsanol::Parser
+  root :document
+  
+  # Introduce a scope for each document. This ensures that documents can be
+  # nested. 
+  rule(:document) { scope { doc_start >> text >> doc_end } }
+  
+  # Start of a document is a heredoc marker. This is captured in :marker
+  rule(:doc_start) { str('<') >> marker >> newline }
+  rule(:marker) { match['A-Z'].repeat(1).capture(:marker) }
+
+  # The content of a document can be either lines of text or another 
+  # document, introduced by <HERE, where HERE is the doc marker. 
+  rule(:text) { (document.as(:doc) | text_line.as(:line)).repeat(1) }
+  rule(:text_line) { captured_marker.absent? >> any >> 
+    (newline.absent? >> any).repeat >> newline }
+    
+  # The end of the document is marked by the marker that was at the beginning
+  # of the document, by itself on a line.
+  rule(:doc_end) { captured_marker }
+  rule(:captured_marker) { 
+    dynamic { |source, context|
+      str(context.captures[:marker])
+    }
+  }
+  
+  rule(:newline) { match["\n"] }
+end
+
+parser = CapturingParser.new
+pp parser.parse_with_debug %Q(<CAPTURE
+Text1
+<FOOBAR
+Text3
+Text4
+FOOBAR
+Text2
+CAPTURE)
+

data/example/capture/basic.rb.md

@@ -0,0 +1,106 @@
+# Capture Patterns - Ruby Implementation
+
+## How to Run
+
+```bash
+cd parsanol-ruby/example/capture
+ruby basic.rb
+```
+
+## Code Walkthrough
+
+### Capture Mechanism
+
+The `capture(:name)` method captures matched text for later reference:
+
+```ruby
+rule(:marker) { match['A-Z'].repeat(1).capture(:marker) }
+```
+
+This captures the heredoc marker (like `CAPTURE` or `FOOBAR`) in the context.
+
+### Dynamic Matching
+
+The `dynamic` block accesses captured values:
+
+```ruby
+rule(:captured_marker) {
+  dynamic { |source, context|
+    str(context.captures[:marker])
+  }
+}
+```
+
+`context.captures[:marker]` returns the previously captured string.
+
+### Scope Isolation
+
+The `scope` block creates isolated capture contexts:
+
+```ruby
+rule(:document) { scope { doc_start >> text >> doc_end } }
+```
+
+Each document has its own capture namespace, allowing nested heredocs.
+
+### Document Structure
+
+Heredoc-style documents with nested content:
+
+```ruby
+rule(:text) {
+  (document.as(:doc) | text_line.as(:line)).repeat(1)
+}
+```
+
+Documents can contain other documents (nested heredocs).
+
+### End Marker Matching
+
+The end marker must match the start marker:
+
+```ruby
+rule(:doc_end) { captured_marker }
+```
+
+This ensures `<FOOBAR` ... `FOOBAR` pairs are correctly matched.
+
+## Output Types
+
+```ruby
+# Input:
+# <CAPTURE
+# Text1
+# <FOOBAR
+# Text3
+# FOOBAR
+# Text2
+# CAPTURE
+
+# Parse tree (simplified):
+{:doc=>[
+  {:line=>"Text1\n"},
+  {:doc=>[
+    {:line=>"Text3\n"}
+  ]},
+  {:line=>"Text2\n"}
+]}
+```
+
+## Design Decisions
+
+### Why Capture Instead of Backreference?
+
+Parslet's capture is more powerful than regex backreferences. It integrates with the parsing context and supports nesting.
+
+### Why Scope Blocks?
+
+Without scopes, nested documents would share the same capture namespace. Scopes isolate each document's captures.
+
+### Why Dynamic Blocks?
+
+Dynamic blocks provide access to the parsing context at parse time. This enables context-dependent matching.
+
+### Ruby-Only Feature
+
+This feature uses Parslet's `capture`, `dynamic`, and `scope` constructs. These have no direct equivalent in Rust and are specific to Parslet's Ruby implementation.

data/example/capture/example.json

@@ -0,0 +1,39 @@
+{
+  "id": "capture",
+  "title": "Capture DSL",
+  "description": "Demonstrate Parslet's capture DSL for binding parse results to variables within the grammar.",
+  "category": "conceptual",
+  "tags": ["capture", "binding", "parslet", "variable"],
+  "difficulty": "intermediate",
+  "concepts": ["capture", "variable binding", "dynamic values", "cross-referencing"],
+
+  "motivation": {
+    "why": "Capture allows binding intermediate parse results to variables for later use in the grammar. This enables cross-referencing and building complex structures from parts.",
+    "useCases": [
+      "Cross-referencing within a grammar",
+      "Building complex structures from parts",
+      "Referencing earlier parse results in later rules"
+    ]
+  },
+
+  "inputFormat": {
+    "description": "Simple expressions that can be captured and referenced.",
+    "examples": [
+      { "input": "a", "description": "Simple character to capture", "valid": true },
+      { "input": "abc", "description": "Multiple characters", "valid": true }
+    ]
+  },
+
+  "outputFormat": {
+    "description": "Captured values bound to variables.",
+    "structure": {
+      "captured_value": { "description": "The value captured by the capture DSL" }
+    }
+  },
+
+  "rubyOnly": true,
+  "parsletCompatible": true,
+  "implementations": {
+    "ruby": { "basic": "basic.rb" }
+  }
+}

data/example/comments/basic.rb

@@ -0,0 +1,35 @@
+# A small example on how to parse common types of comments. The example
+# started out with parser code from Stephen Waits. 
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'pp'
+require 'parsanol/parslet'
+require 'parsanol/convenience'
+
+class ALanguage < Parsanol::Parser
+  root(:lines)
+  
+  rule(:lines) { line.repeat }
+  rule(:line) { spaces >> expression.repeat >> newline }
+  rule(:newline) { str("\n") >> str("\r").maybe }
+  
+  rule(:expression) { (str('a').as(:a) >> spaces).as(:exp) }
+  
+  rule(:spaces) { space.repeat }
+  rule(:space) { multiline_comment | line_comment | str(' ') }
+  
+  rule(:line_comment) { (str('//') >> (newline.absent? >> any).repeat).as(:line) }
+  rule(:multiline_comment) { (str('/*') >> (str('*/').absent? >> any).repeat >> str('*/')).as(:multi) }
+end
+
+code = %q(
+  a
+  // line comment
+  a a a // line comment
+  a /* inline comment */ a 
+  /* multiline
+  comment */
+)
+
+pp ALanguage.new.parse_with_debug(code)