parsanol 3.0.0

data/example/sentence/basic.rb

@@ -0,0 +1,36 @@
+# encoding: UTF-8
+
+# Demonstrates working with Unicode text and natural language.
+# Originally contributed to Parslet, ported to Parsanol as an example.
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'parsanol/parslet'
+
+class MyParser < Parsanol::Parser
+  rule(:sentence) { (match('[^。]').repeat(1) >> str("。")).as(:sentence) }
+  rule(:sentences) { sentence.repeat }
+  root(:sentences)
+end
+
+class Transformer < Parsanol::Transform
+  rule(:sentence => simple(:sen)) { sen.to_s }
+end
+
+string =
+  "RubyKaigi2009のテーマは、「変わる／変える」です。 前回の" +
+  "RubyKaigi2008のテーマであった「多様性」の言葉の通り、 " +
+  "2008年はRubyそのものに関しても、またRubyの活躍する舞台に関しても、 " +
+  "ますます多様化が進みつつあります。RubyKaigi2008は、そのような " +
+  "Rubyの生態系をあらためて認識する場となりました。 しかし、" +
+  "こうした多様化が進む中、異なる者同士が単純に距離を 置いたままでは、" +
+  "その違いを認識したところであまり意味がありません。 異なる実装、" +
+  "異なる思想、異なる背景といった、様々な多様性を理解しつつ、 " +
+  "すり合わせるべきものをすり合わせ、変えていくべきところを " +
+  "変えていくことが、豊かな未来へとつながる道に違いありません。"
+
+parser = MyParser.new
+transformer = Transformer.new
+
+tree = parser.parse(string)
+p transformer.apply(tree)

data/example/sentence/basic.rb.md

@@ -0,0 +1,81 @@
+# Sentence Parser - Ruby Implementation
+
+## How to Run
+
+```bash
+cd parsanol-ruby/example/sentence
+ruby basic.rb
+```
+
+## Code Walkthrough
+
+### Japanese Sentence Rule
+
+Sentences end with the Japanese period character:
+
+```ruby
+rule(:sentence) { (match('[^。]').repeat(1) >> str("。")).as(:sentence) }
+```
+
+The character `。` (U+3002) is the CJK full stop, used as sentence delimiter.
+
+### Multiple Sentences Rule
+
+Text is a sequence of sentences:
+
+```ruby
+rule(:sentences) { sentence.repeat }
+root(:sentences)
+```
+
+Repetition handles arbitrary-length text.
+
+### Transform Rule
+
+The transform extracts sentence content:
+
+```ruby
+class Transformer < Parsanol::Transform
+  rule(:sentence => simple(:sen)) { sen.to_s }
+end
+```
+
+Pattern matching extracts the captured string value.
+
+### Unicode Handling
+
+Ruby 1.9+ handles Unicode natively:
+
+```ruby
+# encoding: UTF-8
+```
+
+The encoding pragma ensures proper interpretation of multibyte characters.
+
+## Output Types
+
+```ruby
+# Parse tree:
+[
+  {:sentence=>"RubyKaigi2009のテーマは、「変わる／変える」です"@s},
+  {:sentence=>" 前回のRubyKaigi2008のテーマであった..."@s}
+]
+
+# After transform:
+["RubyKaigi2009のテーマは、「変わる／変える」です。",
+ " 前回のRubyKaigi2008のテーマであった..."]
+```
+
+## Design Decisions
+
+### Why Use Japanese Period Character?
+
+Natural language parsing must respect language-specific punctuation. Japanese uses `。` not `.`.
+
+### Why Simple Character Class?
+
+`[^。]` excludes only the delimiter. This is simpler than defining valid Japanese character ranges.
+
+### Why Transform to Strings?
+
+Extracting plain strings makes the result easy to process further (count words, analyze sentiment, etc.).

data/example/sexp/ruby_transform.rb

@@ -0,0 +1,180 @@
+# S-Expression Parser Example - RubyTransform
+#
+# This example demonstrates parsing Lisp-style S-expressions.
+# Shows nested lists, atoms, and quoted strings.
+#
+# Run with: ruby -Ilib example/sexp_ruby_transform.rb
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'parsanol'
+
+# Step 1: Define the S-expression grammar
+class SexpParser < Parsanol::Parser
+  root :sexp
+
+  # An S-Expression can be a list or atom
+  rule(:sexp) {
+    list | atom
+  }
+
+  # List: ( ... ) - recursively contains sexps
+  rule(:list) {
+    str('(') >> elements >> str(')')
+  }
+
+  # Elements: zero or more sexps separated by whitespace
+  rule(:elements) {
+    (sexp >> space?).repeat
+  }
+
+  # Atom: number or symbol (whitespace required to separate)
+  rule(:atom) {
+    number | symbol
+  }
+
+  # Symbol: sequence of non-whitespace, non-special chars
+  rule(:symbol) {
+    match('[^\s\(\)]+')
+  }
+
+  # Number: integer or float
+  rule(:number) {
+    (
+      str('-').maybe >>
+      match('[0-9]').repeat(1) >>
+      (str('.') >> match('[0-9]').repeat).maybe
+    )
+  }
+
+  rule(:space?) { match('\s').repeat }
+end
+
+# Step 2: S-expression node classes
+class Sexp; end
+
+class SexpList < Sexp
+  attr_reader :elements
+
+  def initialize(elements)
+    @elements = elements
+  end
+
+  def to_s
+    "(#{@elements.map(&:to_s).join(' ')})"
+  end
+end
+
+class SexpSymbol < Sexp
+  attr_reader :name
+
+  def initialize(name)
+    @name = name.to_s
+  end
+
+  def to_s
+    @name
+  end
+end
+
+class SexpNumber < Sexp
+  attr_reader :value
+
+  def initialize(value)
+    @value = value.to_s
+  end
+
+  def to_s
+    @value
+  end
+end
+
+# Step 3: Parse and transform
+def parse_sexp(input)
+  parser = SexpParser.new
+  tree = parser.parse(input)
+
+  puts "Parse tree: #{tree.inspect}"
+
+  # Build AST
+  ast = build_ast(tree)
+  puts "AST: #{ast.to_s}"
+
+  ast
+rescue Parsanol::ParseFailed => e
+  puts "Parse failed: #{e.message}"
+  nil
+end
+
+def build_ast(tree)
+  return nil if tree.nil?
+
+  # Handle slice
+  if tree.is_a?(Parsanol::Slice)
+    s = tree.to_s
+    if s.match?(/^-?\d+(\.\d+)?$/)
+      SexpNumber.new(s)
+    else
+      SexpSymbol.new(s)
+    end
+  elsif tree.is_a?(Array)
+    # It's a list of sexps
+    elements = tree.map { |t| build_ast(t) }.compact
+    SexpList.new(elements)
+  elsif tree.is_a?(Hash)
+    # Try to find the actual sexp value
+    if tree[:sexp]
+      build_ast(tree[:sexp])
+    elsif tree[:list]
+      build_ast(tree[:list])
+    elsif tree[:elements]
+      build_ast(tree[:elements])
+    elsif tree[:atom]
+      build_ast(tree[:atom])
+    else
+      # Just use the raw string
+      s = tree.to_s
+      if s.match?(/^-?\d+(\.\d+)?$/)
+        SexpNumber.new(s)
+      else
+        SexpSymbol.new(s)
+      end
+    end
+  else
+    # Treat as string
+    s = tree.to_s
+    if s.match?(/^-?\d+(\.\d+)?$/)
+      SexpNumber.new(s)
+    else
+      SexpSymbol.new(s)
+    end
+  end
+end
+
+# Example usage
+if __FILE__ == $0
+  puts "=" * 60
+  puts "S-Expression Parser - RubyTransform"
+  puts "=" * 60
+  puts
+
+  test_cases = [
+    "42",
+    "hello",
+    "(+ 1 2)",
+    "(+ 1 (* 2 3))",
+    "(list 1 2 3)",
+    "()",
+  ]
+
+  test_cases.each do |input|
+    puts "-" * 40
+    puts "Input: #{input}"
+    begin
+      ast = parse_sexp(input)
+    rescue => e
+      puts "Error: #{e.message}"
+    end
+    puts
+  end
+end

data/example/sexp/ruby_transform.rb.md

@@ -0,0 +1,143 @@
+# S-Expression Parser - Ruby Implementation (Transform)
+
+## How to Run
+
+```bash
+cd parsanol-ruby/example/sexp
+ruby ruby_transform.rb
+```
+
+## Code Walkthrough
+
+### S-Expression Structure
+
+An S-expression is either a list or an atom:
+
+```ruby
+rule(:sexp) {
+  list | atom
+}
+```
+
+This recursive definition allows arbitrarily nested structures.
+
+### List Rule
+
+Lists are parenthesized elements:
+
+```ruby
+rule(:list) {
+  str('(') >> elements >> str(')')
+}
+
+rule(:elements) {
+  (sexp >> space?).repeat
+}
+```
+
+Elements are separated by optional whitespace, allowing `(a b c)` or `(a  b    c)`.
+
+### Atom Rule
+
+Atoms are numbers or symbols:
+
+```ruby
+rule(:atom) {
+  number | symbol
+}
+
+rule(:symbol) {
+  match('[^\s\(\)]+')
+}
+
+rule(:number) {
+  (
+    str('-').maybe >>
+    match('[0-9]').repeat(1) >>
+    (str('.') >> match('[0-9]').repeat).maybe
+  )
+}
+```
+
+Symbols exclude whitespace and parentheses; numbers support negative and decimal.
+
+### AST Node Classes
+
+Typed classes represent the parsed structure:
+
+```ruby
+class SexpList < Sexp
+  attr_reader :elements
+  def to_s
+    "(#{@elements.map(&:to_s).join(' ')})"
+  end
+end
+
+class SexpSymbol < Sexp
+  attr_reader :name
+end
+
+class SexpNumber < Sexp
+  attr_reader :value
+end
+```
+
+Each class has a meaningful `to_s` for debugging.
+
+### AST Builder Function
+
+A recursive function builds the AST:
+
+```ruby
+def build_ast(tree)
+  return nil if tree.nil?
+
+  if tree.is_a?(Parsanol::Slice)
+    s = tree.to_s
+    if s.match?(/^-?\d+(\.\d+)?$/)
+      SexpNumber.new(s)
+    else
+      SexpSymbol.new(s)
+    end
+  elsif tree.is_a?(Array)
+    elements = tree.map { |t| build_ast(t) }.compact
+    SexpList.new(elements)
+  # ...
+  end
+end
+```
+
+Pattern matching on tree type determines node construction.
+
+## Output Types
+
+```ruby
+# Input: "(+ 1 (* 2 3))"
+# AST:
+#<SexpList @elements=[
+  #<SexpSymbol @name="+">,
+  #<SexpNumber @value="1">,
+  #<SexpList @elements=[
+    #<SexpSymbol @name="*">,
+    #<SexpNumber @value="2">,
+    #<SexpNumber @value="3">
+  ]>
+]>
+
+# to_s output:
+"(+ 1 (* 2 3))"
+```
+
+## Design Decisions
+
+### Why Recursive AST Builder?
+
+The parse tree has varied structure (Hash, Array, Slice). A recursive function handles all cases uniformly.
+
+### Why Separate Number and Symbol Classes?
+
+Different AST node types allow type-specific behavior (evaluation, formatting, analysis).
+
+### Why Not Use Transform?
+
+The tree structure varies; manual recursion provides more control than pattern-based transforms for this case.

data/example/simple-xml/basic.rb

@@ -0,0 +1,54 @@
+# A simple xml parser. It is simple in the respect as that it doesn't address
+# any of the complexities of XML. This is ruby 1.9.
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'pp'
+require 'parsanol/parslet'
+
+class XML < Parsanol::Parser
+  root :document
+  
+  rule(:document) {
+    tag(close: false).as(:o) >> document.as(:i) >> tag(close: true).as(:c) |
+    text
+  }
+  
+  # Perhaps we could have some syntax sugar to make this more easy?
+  #
+  def tag(opts={})
+    close = opts[:close] || false
+    
+    parslet = str('<')
+    parslet = parslet >> str('/') if close
+    parslet = parslet >> (str('>').absent? >> match("[a-zA-Z]")).repeat(1).as(:name)
+    parslet = parslet >> str('>')
+    
+    parslet
+  end
+  
+  rule(:text) {
+    match('[^<>]').repeat(0)
+  }
+end
+
+def check(xml)
+  r = XML.new.parse(xml)
+
+  # We'll validate the tree by reducing valid pairs of tags into simply the
+  # string "verified". If the transformation ends on a string, then the
+  # document was 'valid'. 
+  #
+  t = Parsanol::Transform.new do
+    rule(
+      o: {name: simple(:tag)}, 
+      c: {name: simple(:tag)}, 
+      i: simple(:t)
+    ) { 'verified' } 
+  end
+
+  t.apply(r)
+end
+
+pp check("<a><b>some text in the tags</b></a>")
+pp check("<b><b>some text in the tags</b></a>")

data/example/simple-xml/basic.rb.md

@@ -0,0 +1,125 @@
+# Simple XML Parser - Ruby Implementation
+
+## How to Run
+
+```bash
+cd parsanol-ruby/example/simple-xml
+ruby basic.rb
+```
+
+## Code Walkthrough
+
+### Document Rule
+
+Documents are nested tag pairs with content:
+
+```ruby
+rule(:document) {
+  tag(close: false).as(:o) >> document.as(:i) >> tag(close: true).as(:c) |
+  text
+}
+```
+
+Recursive structure allows arbitrary nesting depth.
+
+### Tag Rule with Parameter
+
+Tags are generated dynamically based on open/close state:
+
+```ruby
+def tag(opts={})
+  close = opts[:close] || false
+
+  parslet = str('<')
+  parslet = parslet >> str('/') if close
+  parslet = parslet >> (str('>').absent? >> match("[a-zA-Z]")).repeat(1).as(:name)
+  parslet = parslet >> str('>')
+
+  parslet
+end
+```
+
+Method generates opening `<tag>` or closing `</tag>` patterns.
+
+### Text Rule
+
+Text content excludes angle brackets:
+
+```ruby
+rule(:text) {
+  match('[^<>]').repeat(0)
+}
+```
+
+Simple character class prevents tag confusion.
+
+### Tag Validation via Transform
+
+A transform validates matching open/close tags:
+
+```ruby
+t = Parsanol::Transform.new do
+  rule(
+    o: {name: simple(:tag)},
+    c: {name: simple(:tag)},
+    i: simple(:t)
+  ) { 'verified' }
+end
+```
+
+Pattern matching ensures both tags have the same name.
+
+### Validation Logic
+
+If tags don't match, the transform fails:
+
+```ruby
+def check(xml)
+  r = XML.new.parse(xml)
+  t = Parsanol::Transform.new do
+    rule(
+      o: {name: simple(:tag)},
+      c: {name: simple(:tag)},
+      i: simple(:t)
+    ) { 'verified' }
+  end
+  t.apply(r)
+end
+```
+
+Returns 'verified' for valid XML, fails otherwise.
+
+## Output Types
+
+```ruby
+# Valid XML:
+{
+  o: {name: "a"},
+  i: {
+    o: {name: "b"},
+    i: "some text in the tags",
+    c: {name: "b"}
+  },
+  c: {name: "a"}
+}
+
+# After validation:
+"verified"
+
+# Mismatched tags:
+# Transform fails (pattern doesn't match)
+```
+
+## Design Decisions
+
+### Why Validate via Transform?
+
+Transforms can express constraints that are hard to encode in the grammar itself.
+
+### Why Method Instead of Rule for Tags?
+
+`tag(close: true/false)` demonstrates dynamic rule generation, useful for related patterns.
+
+### Why Not Full XML?
+
+This is a teaching example. Real XML requires handling attributes, namespaces, CDATA, etc.

data/example/simple.lit

@@ -0,0 +1,3 @@
+123
+12345
+" Some String with \"escapes\""

data/example/string-literal/basic.rb

@@ -0,0 +1,77 @@
+# A more complex parser that illustrates how a compiler might be constructed.
+# The parser recognizes strings and integer literals and constructs almost a
+# useful AST from the file contents. 
+
+require 'pp'
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require 'parsanol/parslet'
+
+include Parsanol::Parslet
+
+class LiteralsParser < Parsanol::Parser
+  rule :space do
+    (match '[ ]').repeat(1)
+  end
+  
+  rule :literals do
+    (literal >> eol).repeat
+  end
+  
+  rule :literal do
+    (integer | string).as(:literal) >> space.maybe
+  end
+  
+  rule :string do
+    str('"') >> 
+    (
+      (str('\\') >> any) |
+      (str('"').absent? >> any)
+    ).repeat.as(:string) >> 
+    str('"')
+  end
+  
+  rule :integer do
+    match('[0-9]').repeat(1).as(:integer)
+  end
+  
+  rule :eol do
+    line_end.repeat(1)
+  end
+  
+  rule :line_end do
+    crlf >> space.maybe
+  end
+  
+  rule :crlf do
+    match('[\r\n]').repeat(1)
+  end
+
+  root :literals
+end
+
+input_name = File.join(File.dirname(__FILE__), 'simple.lit')
+file = File.read(input_name)
+
+parsetree = LiteralsParser.new.parse(file)
+  
+class Lit < Struct.new(:text)
+  def to_s
+    text.inspect
+  end
+end
+class StringLit < Lit
+end
+class IntLit < Lit
+  def to_s
+    text
+  end
+end
+
+transform = Parsanol::Transform.new do
+  rule(:literal => {:integer => simple(:x)}) { IntLit.new(x) }
+  rule(:literal => {:string => simple(:s)}) { StringLit.new(s) }
+end
+  
+ast = transform.apply(parsetree)
+pp ast