parsanol 3.0.0

data/example/comments/basic.rb.md

@@ -0,0 +1,110 @@
+# Comments Parser - Ruby Implementation
+
+## How to Run
+
+```bash
+cd parsanol-ruby/example/comments
+ruby basic.rb
+```
+
+## Code Walkthrough
+
+### Line Comment Rule
+
+Single-line comments start with `//`:
+
+```ruby
+rule(:line_comment) {
+  (str('//') >> (newline.absent? >> any).repeat).as(:line)
+}
+```
+
+Content extends to end of line; newline is not consumed.
+
+### Multiline Comment Rule
+
+Block comments use `/* */` delimiters:
+
+```ruby
+rule(:multiline_comment) {
+  (str('/*') >> (str('*/').absent? >> any).repeat >> str('*/')).as(:multi)
+}
+```
+
+Negative lookahead prevents early termination.
+
+### Space Rule with Comments
+
+Comments are treated as whitespace:
+
+```ruby
+rule(:spaces) { space.repeat }
+rule(:space) { multiline_comment | line_comment | str(' ') }
+```
+
+This allows comments anywhere whitespace is permitted.
+
+### Expression Rule
+
+Simple expressions demonstrate comment handling:
+
+```ruby
+rule(:expression) { (str('a').as(:a) >> spaces).as(:exp) }
+```
+
+The `spaces` rule consumes any trailing comments.
+
+### Lines and Line Endings
+
+Input is structured as lines:
+
+```ruby
+rule(:lines) { line.repeat }
+rule(:line) { spaces >> expression.repeat >> newline }
+rule(:newline) { str("\n") >> str("\r").maybe }
+```
+
+Each line ends with a newline (CRLF or LF).
+
+### parse_with_debug
+
+Debug output shows the complete parse tree:
+
+```ruby
+pp ALanguage.new.parse_with_debug(code)
+```
+
+Useful for understanding how comments integrate with the grammar.
+
+## Output Types
+
+```ruby
+# Parse tree for:
+#   a // comment
+#   a a a /* inline */ a
+#
+[
+  {:exp=>[{:a=>"a"}]},
+  {:exp=>[{:a=>"a"}, {:a=>"a"}, {:a=>"a"}, {:a=>"a"}]}
+]
+```
+
+Comments are consumed by the `spaces` rule and don't appear in output.
+
+## Design Decisions
+
+### Why Treat Comments as Whitespace?
+
+Comments should be allowed anywhere whitespace is. Making them part of the `space` rule achieves this elegantly.
+
+### Why Not Include Newline in Line Comments?
+
+Newlines are handled separately by the line structure. This keeps comment content clean.
+
+### Why Use Negative Lookahead for Multiline Comments?
+
+`str('*/').absent?` ensures we don't prematurely match the closing delimiter. This is cleaner than trying to enumerate valid characters.
+
+### Why parse_with_debug?
+
+When building grammars, seeing the full parse tree helps debug unexpected matches or failures.

data/example/csv/ruby_transform.rb

@@ -0,0 +1,148 @@
+# CSV Parser Example - Ruby Transform: Ruby Transform (Parslet-Compatible)
+#
+# This example demonstrates Ruby Transform for parsing CSV:
+# 1. Rust parser (parsanol-rs) does the fast parsing
+# 2. Returns a generic tree (hash/array/string structure)
+# 3. Ruby transform converts tree to Ruby objects
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'parsanol'
+
+# Step 1: Define the CSV parser grammar
+class CsvParser < Parsanol::Parser
+  root :csv
+
+  rule(:csv) {
+    space? >> (row >> (newline >> row).repeat).maybe >> space?
+  }
+
+  rule(:row) {
+    (field.as(:f) >> (comma >> field.as(:f)).repeat).as(:row)
+  }
+
+  rule(:field) {
+    quoted_field | simple_field
+  }
+
+  # Quoted field: "value with ""escaped"" quotes"
+  rule(:quoted_field) {
+    str('"') >> (
+      str('""') | str('"').absent? >> any
+    ).repeat.as(:quoted) >> str('"')
+  }
+
+  # Simple field: value without commas or quotes
+  rule(:simple_field) {
+    (comma.absent? >> newline.absent? >> any).repeat.as(:simple)
+  }
+
+  # Helpers
+  rule(:comma) { str(',') }
+  rule(:newline) { str("\n") | str("\r\n") | str("\r") }
+  rule(:space) { match('\s').repeat(1) }
+  rule(:space?) { space.maybe }
+end
+
+# Step 2: Define the transform (Parslet-style)
+class CsvTransform < Parsanol::Transform
+  # Transform a row (sequence of fields)
+  rule(row: sequence(:fields)) {
+    fields.map { |f| f.is_a?(Hash) ? unescape(f) : f }
+  }
+
+  # Transform quoted field
+  rule(quoted: simple(:q)) {
+    q.to_s.gsub('""', '"')
+  }
+
+  # Transform simple field
+  rule(simple: simple(:s)) {
+    s.to_s.strip
+  }
+end
+
+# Step 3: Parse and transform
+def parse_csv(input)
+  parser = CsvParser.new
+  transform = CsvTransform.new
+
+  # Ruby Transform: Parse in Rust, transform in Ruby
+  tree = parser.parse(input)
+  puts "Parse tree: #{tree.inspect[0..200]}..."
+
+  result = transform.apply(tree)
+  puts "Result: #{result.inspect[0..200]}..."
+
+  result
+end
+
+# Step 4: Convert to array of hashes (for CSV with headers)
+def parse_csv_with_headers(input)
+  rows = parse_csv(input)
+
+  return [] if rows.empty?
+
+  # First row is headers
+  headers = rows.first
+  data = rows[1..]
+
+  data.map { |row| headers.zip(row).to_h }
+end
+
+# Example usage
+if __FILE__ == $0
+  puts "=" * 60
+  puts "CSV Parser Example - Ruby Transform: Ruby Transform"
+  puts "=" * 60
+
+  # Simple CSV
+  simple_csv = <<~CSV
+    name,age,city
+    Alice,30,New York
+    Bob,25,San Francisco
+  CSV
+
+  puts
+  puts "Simple CSV:"
+  puts "-" * 40
+  result = parse_csv(simple_csv)
+  puts result.inspect
+
+  # CSV with headers parsed to hashes
+  puts
+  puts "CSV with headers:"
+  puts "-" * 40
+  result = parse_csv_with_headers(simple_csv)
+  result.each { |row| puts row.inspect }
+
+  # CSV with quoted fields
+  quoted_csv = <<~CSV
+    name,description,city
+    Alice,"Hello, World",New York
+    Bob,"Test ""quoted"" text",Boston
+  CSV
+
+  puts
+  puts "CSV with quoted fields:"
+  puts "-" * 40
+  result = parse_csv_with_headers(quoted_csv)
+  result.each { |row| puts row.inspect }
+
+  # Empty CSV
+  empty_csv = ""
+
+  puts
+  puts "Empty CSV:"
+  puts "-" * 40
+  result = parse_csv(empty_csv)
+  puts result.inspect
+
+  puts
+  puts "=" * 60
+  puts "Ruby Transform Benefits for CSV:"
+  puts "- Flexible transform logic"
+  puts "- Easy to add custom processing"
+  puts "- Compatible with existing Parslet code"
+  puts "=" * 60
+end

data/example/csv/ruby_transform.rb.md

@@ -0,0 +1,131 @@
+# CSV Parser - Ruby Implementation (Transform)
+
+## How to Run
+
+```bash
+cd parsanol-ruby/example/csv
+ruby ruby_transform.rb
+```
+
+## Code Walkthrough
+
+### CSV Grammar Definition
+
+The grammar handles rows, fields, and quoted content:
+
+```ruby
+rule(:csv) {
+  space? >> (row >> (newline >> row).repeat).maybe >> space?
+}
+
+rule(:row) {
+  (field.as(:f) >> (comma >> field.as(:f)).repeat).as(:row)
+}
+```
+
+Each row captures multiple fields labeled `:f`, wrapped in `:row`.
+
+### Quoted Field Handling
+
+Quoted fields support escaped quotes:
+
+```ruby
+rule(:quoted_field) {
+  str('"') >> (
+    str('""') | str('"').absent? >> any
+  ).repeat.as(:quoted) >> str('"')
+}
+```
+
+Double quotes (`""`) inside quoted fields represent literal quotes.
+
+### Simple Field Handling
+
+Simple fields exclude commas and newlines:
+
+```ruby
+rule(:simple_field) {
+  (comma.absent? >> newline.absent? >> any).repeat.as(:simple)
+}
+```
+
+Negative lookahead prevents field content from including delimiters.
+
+### Field Rule Selection
+
+The field rule tries quoted first:
+
+```ruby
+rule(:field) {
+  quoted_field | simple_field
+}
+```
+
+Quoted fields have priority to correctly handle fields starting with `"`.
+
+### Transform Rules
+
+The transform converts parse tree to Ruby arrays:
+
+```ruby
+class CsvTransform < Parsanol::Transform
+  # Transform a row (sequence of fields)
+  rule(row: sequence(:fields)) {
+    fields.map { |f| f.is_a?(Hash) ? unescape(f) : f }
+  }
+
+  # Transform quoted field
+  rule(quoted: simple(:q)) {
+    q.to_s.gsub('""', '"')
+  }
+
+  # Transform simple field
+  rule(simple: simple(:s)) {
+    s.to_s.strip
+  }
+end
+```
+
+Pattern matching extracts field content and converts to strings.
+
+### Header-Based Parsing
+
+CSV with headers converts to array of hashes:
+
+```ruby
+def parse_csv_with_headers(input)
+  rows = parse_csv(input)
+  return [] if rows.empty?
+
+  headers = rows.first
+  data = rows[1..]
+
+  data.map { |row| headers.zip(row).to_h }
+end
+```
+
+First row becomes keys; subsequent rows become values.
+
+## Output Types
+
+```ruby
+# Without headers:
+[["name", "age", "city"], ["Alice", "30", "New York"], ...]
+
+# With headers:
+[{"name"=>"Alice", "age"=>"30", "city"=>"New York"}, ...]
+```
+
+## Design Decisions
+
+### Why Ruby Transform Over Rust?
+
+Ruby transform allows custom processing logic without modifying Rust code. Useful for domain-specific transformations and data enrichment.
+
+### Why Sequence Pattern for Rows?
+
+`sequence(:fields)` handles both single and multiple fields uniformly, avoiding special cases for one-field rows.
+
+### Why Priority for Quoted Fields?
+
+If simple field were first, `"hello"` would match as simple field `"` followed by errors. Quoted field priority ensures correct parsing.

data/example/csv/serialized.rb

@@ -0,0 +1,201 @@
+# CSV Parser Example - Serialized: JSON Serialization
+#
+# This example demonstrates Serialized for parsing CSV:
+# 1. Rust parser (parsanol-rs) does the parsing
+# 2. Rust transform converts to typed structs
+# 3. Result is serialized to JSON
+# 4. Ruby deserializes JSON to Ruby objects
+#
+# This option is useful when you need to validate/proces CSV
+# and get structured output for other tools.
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'parsanol'
+require 'json'
+
+# NOTE: This example requires the native extension to support parse_to_json
+# which is planned but not yet implemented. This serves as an API preview.
+
+# Step 1: Define the CSV parser grammar (same as Option A)
+class CsvParser < Parsanol::Parser
+  root :csv
+
+  rule(:csv) {
+    space? >> (row >> (newline >> row).repeat).maybe >> space?
+  }
+
+  rule(:row) {
+    (field.as(:f) >> (comma >> field.as(:f)).repeat).as(:row)
+  }
+
+  rule(:field) {
+    quoted_field | simple_field
+  }
+
+  rule(:quoted_field) {
+    str('"') >> (
+      str('""') | str('"').absent? >> any
+    ).repeat.as(:quoted) >> str('"')
+  }
+
+  rule(:simple_field) {
+    (comma.absent? >> newline.absent? >> any).repeat.as(:simple)
+  }
+
+  rule(:comma) { str(',') }
+  rule(:newline) { str("\n") | str("\r\n") | str("\r") }
+  rule(:space) { match('\s').repeat(1) }
+  rule(:space?) { space.maybe }
+end
+
+# Step 2: Define typed classes for CSV data
+class CsvRow
+  attr_reader :fields
+
+  def initialize(fields)
+    @fields = fields
+  end
+
+  def to_a = @fields
+
+  def [](index)
+    @fields[index]
+  end
+
+  def each(&block)
+    @fields.each(&block)
+  end
+end
+
+class CsvDocument
+  attr_reader :rows
+
+  def initialize(rows)
+    @rows = rows
+  end
+
+  def to_a
+    @rows.map(&:to_a)
+  end
+
+  def headers
+    @rows.first&.fields
+  end
+
+  def data
+    @rows[1..] || []
+  end
+
+  def to_hashes
+    return [] unless headers && !data.empty?
+
+    headers = self.headers
+    data.map { |row| headers.zip(row.fields).to_h }
+  end
+end
+
+# Step 3: Deserializer
+class CsvDeserializer
+  def self.from_json(json_string)
+    data = JSON.parse(json_string)
+
+    case data
+    when Array
+      rows = data.map { |row_data| CsvRow.new(row_data) }
+      CsvDocument.new(rows)
+    else
+      raise "Expected array of rows, got #{data.class}"
+    end
+  end
+end
+
+# Step 4: Parse with JSON output
+def parse_csv(input)
+  parser = CsvParser.new
+
+  # Serialized: Parse and get JSON from Rust
+  # NOTE: This requires native extension support
+  # output_json = parser.parse_to_json(input)
+
+  # For now, simulate by using Option A then serializing
+  require_relative 'csv_option_a'
+  tree = parser.parse(input)
+  transform = CsvTransform.new
+  result = transform.apply(tree)
+
+  # This would come from Rust in Serialized
+  # Convert to array format for JSON
+  output_json = result.to_json
+  puts "Output JSON (first 200 chars): #{output_json[0..200]}..."
+
+  # Deserialize to typed objects
+  csv_doc = CsvDeserializer.from_json(output_json)
+  puts "Parsed: #{csv_doc.class} with #{csv_doc.rows.size} rows"
+
+  csv_doc
+end
+
+# Transform class (needed for simulation)
+class CsvTransform < Parsanol::Transform
+  rule(row: sequence(:fields)) {
+    fields.map { |f| f.is_a?(Hash) ? unescape(f) : f }
+  }
+
+  rule(quoted: simple(:q)) { unescape_quoted(q.to_s) }
+  rule(simple: simple(:s)) { s.to_s.strip }
+
+  private
+
+  def unescape(field)
+    if field.is_a?(Hash) && field[:quoted]
+      unescape_quoted(field[:quoted])
+    elsif field.is_a?(Hash) && field[:simple]
+      field[:simple].to_s.strip
+    else
+      field
+    end
+  end
+
+  def unescape_quoted(str)
+    str.gsub('""', '"')
+  end
+end
+
+# Example usage
+if __FILE__ == $0
+  puts "=" * 60
+  puts "CSV Parser Example - Serialized: JSON Serialization"
+  puts "=" * 60
+  puts
+  puts "NOTE: This example shows the planned API for Serialized."
+  puts "The native extension support for parse_to_json is coming soon."
+  puts
+
+  simple_csv = <<~CSV
+    name,age,city
+    Alice,30,New York
+    Bob,25,San Francisco
+  CSV
+
+  puts "Simple CSV:"
+  puts "-" * 40
+  csv_doc = parse_csv(simple_csv)
+
+  puts
+  puts "As arrays:"
+  csv_doc.to_a.each { |row| puts row.inspect }
+
+  puts
+  puts "As hashes:"
+  csv_doc.to_hashes.each { |row| puts row.inspect }
+
+  puts
+  puts "=" * 60
+  puts "Serialized Benefits for CSV:"
+  puts "- Structured JSON output for other tools"
+  puts "- Easy to cache/store results"
+  puts "- Type-safe access via CsvRow/CsvDocument classes"
+  puts "- Cross-language compatibility"
+  puts "=" * 60
+end

data/example/csv/serialized.rb.md

@@ -0,0 +1,31 @@
+# CSV (Serialized - Option B)
+
+## Purpose
+
+This implementation demonstrates full Rust processing with JSON output
+for CSV parsing.
+
+## When to Use
+
+- Cross-language compatibility
+- Structured output required
+- Performance-critical applications
+
+## Key Concepts
+
+1. **Rust Parsing + Transform**: All processing in Rust
+2. **JSON Serialization**: Language-agnostic output
+3. **Type Safety**: Schema-driven structure
+
+## Running
+
+```bash
+ruby example/csv/serialized.rb
+```
+
+## Output
+
+```
+Input: a,b,c
+JSON: [["a","b","c"]]
+```