parsanol 3.0.0

data/example/csv/zero_copy.rb

@@ -0,0 +1,276 @@
+# CSV Parser Example - ZeroCopy: Mirrored Objects (Direct FFI)
+#
+# This example demonstrates ZeroCopy for parsing CSV:
+# 1. Rust parser (parsanol-rs) does the parsing
+# 2. Rust constructs typed CSV value objects
+# 3. Direct Ruby object construction via FFI (no serialization!)
+# 4. Maximum performance with zero-copy
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'parsanol'
+
+# NOTE: This example requires:
+# 1. ZeroCopy extension support for parse_to_objects
+# 2. #[derive(RubyObject)] proc macro in Rust
+# 3. Matching Ruby class definitions
+#
+# This serves as an API preview.
+
+# Step 1: Define Ruby classes that mirror Rust struct definitions
+module Csv
+  class Value
+  end
+
+  # Represents a single CSV field
+  class Field < Value
+    attr_reader :raw, :value
+
+    def initialize(raw:, value:)
+      @raw = raw
+      @value = value
+    end
+
+    def to_s = @value
+    def ==(other)
+      other.is_a?(Field) && @value == other.value
+    end
+  end
+
+  # Represents a row of fields
+  class Row < Value
+    attr_reader :fields
+
+    def initialize(fields)
+      @fields = fields
+    end
+
+    def [](index)
+      @fields[index]
+    end
+
+    def size
+      @fields.size
+    end
+
+    def each(&block)
+      @fields.each(&block)
+    end
+
+    def to_a
+      @fields.map(&:value)
+    end
+
+    def to_s
+      @fields.map(&:value).join(',')
+    end
+  end
+
+  # Represents an entire CSV document
+  class Document < Value
+    attr_reader :rows
+
+    def initialize(rows)
+      @rows = rows
+    end
+
+    def size
+      @rows.size
+    end
+
+    def [](index)
+      @rows[index]
+    end
+
+    def each(&block)
+      @rows.each(&block)
+    end
+
+    def headers
+      @rows.first&.fields&.map(&:value) if @rows.any?
+    end
+
+    def data
+      @rows[1..] || []
+    end
+
+    def to_a
+      @rows.map(&:to_a)
+    end
+
+    def to_hashes
+      return [] unless headers && !data.empty?
+
+      headers = self.headers
+      data.map { |row| headers.zip(row.to_a).to_h }
+    end
+  end
+end
+
+# Step 2: Define the parser with output type mapping
+class CsvParser < Parsanol::Parser
+  # Include ZeroCopy module (planned)
+  # include Parsanol::ZeroCopy
+
+  root :csv
+
+  rule(:csv) {
+    space? >> (row >> (newline >> row).repeat).maybe >> space?
+  }
+
+  rule(:row) {
+    (field.as(:f) >> (comma >> field.as(:f)).repeat).as(:row)
+  }
+
+  rule(:field) {
+    quoted_field | simple_field
+  }
+
+  rule(:quoted_field) {
+    str('"') >> (
+      str('""') | str('"').absent? >> any
+    ).repeat.as(:quoted) >> str('"')
+  }
+
+  rule(:simple_field) {
+    (comma.absent? >> newline.absent? >> any).repeat.as(:simple)
+  }
+
+  rule(:comma) { str(',') }
+  rule(:newline) { str("\n") | str("\r\n") | str("\r") }
+  rule(:space) { match('\s').repeat(1) }
+  rule(:space?) { space.maybe }
+
+  # Output type mapping (planned feature)
+  # output_types(
+  #   field: Csv::Field,
+  #   row: Csv::Row,
+  #   csv: Csv::Document
+  # )
+end
+
+# Step 3: Parse with direct object construction
+def parse_csv(input)
+  parser = CsvParser.new
+
+  # ZeroCopy: Parse and get direct Ruby objects
+  # NOTE: This requires native extension support
+  # doc = parser.parse(input)
+  # # doc is already a Csv::Document!
+  # # No transform needed, no JSON serialization!
+
+  # For demonstration, simulate what ZeroCopy would return
+  doc = simulate_parse(input)
+  puts "Parsed: #{doc.class} with #{doc.size} rows"
+
+  doc
+end
+
+# Simulated parsing for demonstration
+def simulate_parse(input)
+  lines = input.strip.split("\n")
+  return Csv::Document.new([]) if lines.empty?
+
+  rows = lines.map do |line|
+    fields = line.split(',').map do |field|
+      raw = field
+      # Unescape if quoted
+      value = if field.start_with?('"') && field.end_with?('"')
+                field[1..-2].gsub('""', '"')
+              else
+                field.strip
+              end
+      Csv::Field.new(raw: raw, value: value)
+    end
+    Csv::Row.new(fields)
+  end
+
+  Csv::Document.new(rows)
+end
+
+# Example usage
+if __FILE__ == $0
+  puts "=" * 60
+  puts "CSV Parser Example - ZeroCopy: Mirrored Objects"
+  puts "=" * 60
+  puts
+  puts "NOTE: This example shows the planned API for ZeroCopy."
+  puts "The native extension support for parse_to_objects is coming soon."
+  puts
+
+  simple_csv = <<~CSV
+    name,age,city
+    Alice,30,New York
+    Bob,25,San Francisco
+  CSV
+
+  puts "Simple CSV:"
+  puts "-" * 40
+  doc = parse_csv(simple_csv)
+
+  puts "As arrays:"
+  doc.to_a.each { |row| puts row.inspect }
+
+  puts
+  puts "Headers: #{doc.headers.inspect}"
+  puts "Data rows: #{doc.data.size}"
+
+  puts
+  puts "As hashes:"
+  doc.to_hashes.each { |row| puts row.inspect }
+
+  # Type-safe access
+  puts
+  puts "Type-safe access:"
+  puts "First row class: #{doc[0].class}"
+  puts "First field class: #{doc[0][0].class}"
+  puts "First field raw: #{doc[0][0].raw.inspect}"
+  puts "First field value: #{doc[0][0].value.inspect}"
+
+  # Custom method example
+  puts
+  puts "Custom method on Field:"
+  field = Csv::Field.new(raw: '"Hello, World"', value: 'Hello, World')
+  puts "Field: #{field.value}"
+
+  puts
+  puts "=" * 60
+  puts "ZeroCopy Benefits for CSV:"
+  puts "- FASTEST: No serialization overhead"
+  puts "- Type-safe: Each field is a Csv::Field object"
+  puts "- Custom methods: Can add validation, formatting, etc."
+  puts "- Zero-copy: Direct construction from Rust"
+  puts
+  puts "When to use ZeroCopy for CSV:"
+  puts "- High-throughput CSV processing"
+  puts "- When you need typed field access"
+  puts "- When you want custom methods on fields/rows"
+  puts "- When performance is critical"
+  puts "=" * 60
+end
+
+# Rust code that would be needed (for reference):
+#
+# // In parsanol-rs
+# use parsanol_ruby_derive::RubyObject;
+#
+# #[derive(Debug, Clone, RubyObject)]
+# #[ruby_class("Csv::Value")]
+# pub enum CsvValue {
+#     #[ruby_variant("field")]
+#     Field {
+#         raw: String,
+#         value: String,
+#     },
+#
+#     #[ruby_variant("row")]
+#     Row(Vec<CsvValue>),
+#
+#     #[ruby_variant("document")]
+#     Document(Vec<CsvValue>),
+# }
+#
+# // The #[derive(RubyObject)] proc macro generates:
+# // - Ruby class definitions
+# // - to_ruby() implementation
+# // - Direct object construction via FFI

data/example/csv/zero_copy.rb.md

@@ -0,0 +1,36 @@
+# CSV (Zero-Copy - Option C)
+
+## Purpose
+
+This implementation demonstrates direct FFI object construction for CSV
+parsing without serialization overhead.
+
+## When to Use
+
+- Maximum performance required
+- Production systems
+- When zero-copy is critical
+
+## Key Concepts
+
+1. **Direct FFI**: No serialization overhead
+2. **Ruby Object Construction**: Direct via rb_funcall
+3. **Type Safety**: Mirrored types on both sides
+
+## Running
+
+```bash
+ruby example/csv/zero_copy.rb
+```
+
+## Output
+
+```
+Input: a,b,c
+Result: Array
+Value: [["a", "b", "c"]]
+```
+
+## Note
+
+This is the fastest option but requires more complex FFI setup.

data/example/custom_atoms/indent_atom.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# IndentAtom - Custom atom for indentation-sensitive matching
+#
+# This example demonstrates how to create a custom atom that matches
+# lines with a specific indentation level. This is useful for parsing
+# Python-like languages where indentation matters.
+#
+# Usage:
+#   class PythonParser < Parsanol::Parser
+#     rule(:indented_block) { IndentAtom.new(4) >> statement }
+#   end
+
+require 'parsanol'
+
+class IndentAtom < Parsanol::Atoms::Custom
+  # Create a new indentation matcher
+  #
+  # @param expected_indent [Integer] Number of spaces expected at start of line
+  def initialize(expected_indent)
+    @expected_indent = expected_indent
+    super()
+  end
+
+  # Required: Implement the matching logic
+  def try_match(source, context, consume_all)
+    # Save position for potential backtrack
+    start_pos = source.bytepos
+
+    # Count leading spaces at current position
+    indent = 0
+    while source.bytepos < source.chars_left + start_pos
+      # Peek at the next character without consuming it
+      break unless source.matches?(/ /)
+
+      # Consume one space
+      char = source.consume(1)
+      break unless char == ' '
+      indent += 1
+    end
+
+    if indent == @expected_indent
+      # Success - return the matched indentation as a slice
+      # Use source.slice to create a proper Slice object
+      [true, source.slice(start_pos, ' ' * indent)]
+    else
+      # Failure - restore position for backtracking
+      source.bytepos = start_pos
+      [false, nil]
+    end
+  end
+
+  # Override to_s_inner for better error messages
+  def to_s_inner(prec = nil)
+    "indent(#{@expected_indent})"
+  end
+end
+
+# Example usage
+if __FILE__ == $0
+  class IndentedParser < Parsanol::Parser
+    rule(:line) { indent >> content }
+    rule(:indent) { IndentAtom.new(2) }
+    rule(:content) { match['a-z'].repeat(1) }
+    root(:line)
+  end
+
+  parser = IndentedParser.new
+
+  # This should parse - exactly 2 spaces of indentation
+  puts parser.parse("  hello").inspect
+
+  # This will fail - wrong indentation
+  begin
+    puts parser.parse("    hello").inspect
+  rescue Parsanol::ParseFailed => e
+    puts "Failed as expected: #{e.message}"
+  end
+end

data/example/deepest-errors/basic.rb

@@ -0,0 +1,131 @@
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+# This example demonstrates how to do deepest error reporting, as invented 
+# by John Mettraux (issue #64).
+
+require 'parsanol/parslet'
+require 'parsanol/convenience'
+
+def prettify(str)
+  puts " "*3 + " "*4 + "." + " "*4 + "10" + " "*3 + "." + " "*4 + "20"
+  str.lines.each_with_index do |line, index|
+    printf "%02d %s\n", 
+      index+1, 
+      line.chomp
+  end
+end
+
+class MyParser < Parsanol::Parser
+  # commons
+
+  rule(:space) { match('[ \t]').repeat(1) }
+  rule(:space?) { space.maybe }
+
+  rule(:newline) { match('[\r\n]') }
+
+  rule(:comment) { str('#') >> match('[^\r\n]').repeat }
+
+  rule(:line_separator) {
+    (space? >> ((comment.maybe >> newline) | str(';')) >> space?).repeat(1)
+  }
+
+  rule(:blank) { line_separator | space }
+  rule(:blank?) { blank.maybe }
+
+  rule(:identifier) { match('[a-zA-Z0-9_]').repeat(1) }
+
+  # res_statement
+
+  rule(:reference) {
+    (str('@').repeat(1,2) >> identifier).as(:reference)
+  }
+
+  rule(:res_action_or_link) {
+    str('.').as(:dot) >> (identifier >> str('?').maybe ).as(:name) >> str('()')
+  }
+
+  rule(:res_actions) {
+    (
+      reference
+    ).as(:resources) >>
+    (
+      res_action_or_link.as(:res_action)
+    ).repeat(0).as(:res_actions)
+  }
+
+  rule(:res_statement) {
+    res_actions >>
+    (str(':') >> identifier.as(:name)).maybe.as(:res_field)
+  }
+
+  # expression
+
+  rule(:expression) {
+    res_statement
+  }
+
+  # body
+
+  rule(:body) {
+    (line_separator >> (block | expression)).repeat(1).as(:body) >>
+    line_separator
+  }
+
+  # blocks
+
+  rule(:begin_block) {
+    (str('concurrent').as(:type) >> space).maybe.as(:pre) >>
+    str('begin').as(:begin) >>
+    body >>
+    str('end')
+  }
+
+  rule(:define_block) {
+    str('define').as(:define) >> space >>
+    identifier.as(:name) >> str('()') >>
+    body >>
+    str('end')
+  }
+
+  rule(:block) {
+    define_block | begin_block
+  }
+
+  # root
+
+  rule(:radix) {
+    line_separator.maybe >> block >> line_separator.maybe
+  }
+
+  root(:radix)
+end
+
+ds = [
+  %{
+    define f()
+      @res.name
+    end
+  },
+  %{
+    define f()
+      begin
+        @res.name
+      end
+    end
+  }
+]
+
+ds.each do |d|
+
+  puts '-' * 80
+  prettify(d)
+
+  parser = MyParser.new
+
+  begin
+    parser.parse_with_debug(d, 
+      :reporter => Parsanol::ErrorReporter::Deepest.new)
+  end
+end
+
+puts '-' * 80

data/example/deepest-errors/basic.rb.md

@@ -0,0 +1,152 @@
+# Deepest Error Reporting - Ruby Implementation
+
+## How to Run
+
+```bash
+cd parsanol-ruby/example/deepest-errors
+ruby basic.rb
+```
+
+## Code Walkthrough
+
+### Error Reporter Configuration
+
+Parslet provides different error reporters:
+
+```ruby
+parser.parse_with_debug(input,
+  :reporter => Parsanol::ErrorReporter::Deepest.new)
+```
+
+The `Deepest` reporter finds the point where parsing progressed furthest.
+
+### Helper for Display
+
+Format input with line numbers:
+
+```ruby
+def prettify(str)
+  puts " "*3 + " "*4 + "." + " "*4 + "10" + " "*3 + "." + " "*4 + "20"
+  str.lines.each_with_index do |line, index|
+    printf "%02d %s\n", index+1, line.chomp
+  end
+end
+```
+
+This helps users locate errors in their input.
+
+### Complex Grammar
+
+The example uses a realistic grammar with multiple constructs:
+
+```ruby
+rule(:define_block) {
+  str('define').as(:define) >> space >>
+  identifier.as(:name) >> str('()') >>
+  body >>
+  str('end')
+}
+
+rule(:begin_block) {
+  (str('concurrent').as(:type) >> space).maybe.as(:pre) >>
+  str('begin').as(:begin) >>
+  body >>
+  str('end')
+}
+```
+
+Multiple block types demonstrate error scenarios.
+
+### Reference Parsing
+
+Resources use dot notation:
+
+```ruby
+rule(:reference) {
+  (str('@').repeat(1,2) >> identifier).as(:reference)
+}
+
+rule(:res_action_or_link) {
+  str('.').as(:dot) >> (identifier >> str('?').maybe).as(:name) >> str('()')
+}
+```
+
+Single `@` for reference, `@@` for class reference.
+
+### Body Structure
+
+Bodies contain nested content:
+
+```ruby
+rule(:body) {
+  (line_separator >> (block | expression)).repeat(1).as(:body) >>
+  line_separator
+}
+```
+
+Recursion allows arbitrarily nested blocks.
+
+### Testing with Errors
+
+```ruby
+ds = [
+  %{
+    define f()
+      @res.name
+    end
+  },
+  %{
+    define f()
+      begin
+        @res.name
+      end
+    end
+  }
+]
+
+ds.each do |d|
+  parser.parse_with_debug(d, :reporter => Parsanol::ErrorReporter::Deepest.new)
+end
+```
+
+Each test case shows how errors are reported.
+
+## Output Types
+
+```
+01
+02 define f()
+03   @res.name
+04 end
+
+Parsed successfully
+```
+
+Or for errors:
+
+```
+01
+02 define f()
+03   @res.name(
+04 end
+
+Expected ')' at line 3, column 12
+```
+
+## Design Decisions
+
+### Why Deepest Reporter?
+
+The deepest failure point is usually where the user made their mistake. Earlier failures are often from trying wrong alternatives.
+
+### Why parse_with_debug?
+
+This method automatically formats and prints errors. For programmatic access, use `parse` with rescue.
+
+### Why Line Numbers?
+
+Humans think in lines, not byte positions. Line-oriented output matches how users read their input.
+
+### Why Multiple Test Cases?
+
+Different error scenarios demonstrate the reporter's behavior across grammar constructs.

data/example/documentation/basic.rb

@@ -0,0 +1,18 @@
+# A small example that shows a really small parser and what happens on parser
+# errors. 
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'pp'
+require 'parsanol/parslet'
+
+class MyParser < Parsanol::Parser
+  rule(:a) { str('a').repeat }
+  
+  def parse(str)
+    a.parse(str)
+  end
+end
+
+pp MyParser.new.parse('aaaa')
+pp MyParser.new.parse('bbbb')