parsanol 3.0.0

data/example/yaml/basic.rb

@@ -0,0 +1,216 @@
+# YAML Parser - Ruby Implementation
+#
+# Parse a subset of YAML: key-value pairs, lists, nested maps, scalars.
+#
+# Run with: ruby example/yaml/basic.rb
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'parsanol/parslet'
+
+# YAML parser (subset)
+class YamlParser < Parsanol::Parser
+  root :document
+
+  # Document is a sequence of mappings or list items
+  rule(:document) { (mapping | list_item | comment | blank_line).repeat(1).as(:document) }
+
+  # Comment: # to end of line
+  rule(:comment) {
+    (space? >> str('#') >> (newline.absent? >> any).repeat).as(:comment) >> newline
+  }
+
+  # Blank line
+  rule(:blank_line) { space? >> newline }
+
+  # Mapping (key-value)
+  rule(:mapping) {
+    (key.as(:key) >>
+     colon >>
+     (inline_value | indented_value)).as(:mapping)
+  }
+
+  rule(:key) {
+    (match('[a-zA-Z_]') >> match('[a-zA-Z0-9_]').repeat).as(:key)
+  }
+
+  rule(:colon) { str(':') >> space }
+
+  # Inline value: on same line as key
+  rule(:inline_value) {
+    (space? >> (string | integer | float | boolean | null).as(:value) >> newline)
+  }
+
+  # Indented value: nested block
+  rule(:indented_value) {
+    (newline >> indented_block.as(:block))
+  }
+
+  rule(:indented_block) {
+    (indent >> (mapping | list_item) >> (newline >> indent >> (mapping | list_item)).repeat).as(:block)
+  }
+
+  # List item: - value
+  rule(:list_item) {
+    (str('-') >> space >>
+     (inline_list_value | indented_value)).as(:list_item)
+  }
+
+  rule(:inline_list_value) {
+    (space? >> (string | integer | float | boolean | null).as(:value) >> newline)
+  }
+
+  # Scalar types
+  rule(:string) {
+    quoted_string | plain_string
+  }
+
+  rule(:quoted_string) {
+    (str('"') >>
+     (str('\\').ignore >> any | str('"').absent? >> any).repeat.as(:string) >>
+     str('"')) |
+    (str("'") >>
+     (str("'").absent? >> any).repeat.as(:string) >>
+     str("'"))
+  }
+
+  rule(:plain_string) {
+    (newline.absent? >> str(':').absent? >> any).repeat(1).as(:string)
+  }
+
+  rule(:integer) {
+    (str('+') | str('-')).maybe >>
+    match('[0-9]').repeat(1)
+  }
+
+  rule(:float) {
+    ((str('+') | str('-')).maybe >>
+     match('[0-9]').repeat(1) >>
+     str('.') >>
+     match('[0-9]').repeat(1))
+  }
+
+  rule(:boolean) {
+    str('true') | str('false')
+  }
+
+  rule(:null) {
+    str('null') | str('~')
+  }
+
+  # Helpers
+  rule(:space) { match('\s').repeat(1) }
+  rule(:space?) { match('\s').repeat }
+  rule(:newline) { match('\n') | match('\r\n') }
+  rule(:indent) { str('  ') | str("\t") }
+end
+
+# YAML result classes
+YamlDocument = Struct.new(:entries) do
+  def to_h
+    result = {}
+    entries.each do |entry|
+      case entry
+      when YamlMapping
+        result[entry.key] = entry.value
+      end
+    end
+    result
+  end
+end
+
+YamlMapping = Struct.new(:key, :value)
+YamlListItem = Struct.new(:value)
+YamlComment = Struct.new(:text)
+
+# Transform parse tree to AST
+class YamlTransform < Parsanol::Transform
+  rule(document: sequence(:entries)) { YamlDocument.new(entries) }
+
+  rule(mapping: { key: simple(:k), value: simple(:v) }) {
+    YamlMapping.new(k.to_s, v)
+  }
+
+  rule(mapping: { key: simple(:k), block: simple(:b) }) {
+    YamlMapping.new(k.to_s, b)
+  }
+
+  rule(list_item: { value: simple(:v) }) {
+    YamlListItem.new(v)
+  }
+
+  rule(list_item: { block: simple(:b) }) {
+    YamlListItem.new(b)
+  }
+
+  rule(comment: simple(:c)) { YamlComment.new(c.to_s) }
+
+  # Value transformations
+  rule(string: simple(:s)) { s.to_s.strip }
+  rule(integer: simple(:i)) { i.to_s.to_i }
+  rule(float: simple(:f)) { f.to_s.to_f }
+  rule(value: simple(:v)) { v }
+  rule(block: simple(:b)) { b }
+  rule(block: sequence(:bs)) {
+    result = {}
+    bs.each do |b|
+      if b.is_a?(YamlMapping)
+        result[b.key] = b.value
+      end
+    end
+    result
+  }
+end
+
+# Parse YAML string
+def parse_yaml(str)
+  parser = YamlParser.new
+  transform = YamlTransform.new
+
+  tree = parser.parse(str)
+  transform.apply(tree)
+rescue Parsanol::ParseError => e
+  puts "Parse error: #{e.message}"
+  nil
+end
+
+# Main demo
+if __FILE__ == $0
+  puts "YAML Parser"
+  puts "=" * 50
+  puts
+
+  yaml = <<~YAML
+    # Configuration file
+    name: Example Application
+    version: 1.0.0
+    debug: true
+    timeout: 30.5
+
+    database:
+      host: localhost
+      port: 5432
+      name: myapp
+
+    servers:
+      - alpha
+      - beta
+      - gamma
+  YAML
+
+  puts "Input:"
+  puts "-" * 50
+  puts yaml
+  puts "-" * 50
+  puts
+
+  result = parse_yaml(yaml)
+
+  if result
+    puts "Parsed AST:"
+    pp result
+    puts
+    puts "Hash Output:"
+    pp result.to_h
+  end
+end

data/example/yaml/basic.rb.md

@@ -0,0 +1,148 @@
+# YAML Parser - Ruby Implementation
+
+## How to Run
+
+```bash
+cd parsanol-ruby/example/yaml
+ruby basic.rb
+```
+
+## Code Walkthrough
+
+### Document Structure
+
+A YAML document contains mappings, list items, and comments:
+
+```ruby
+rule(:document) { (mapping | list_item | comment | blank_line).repeat(1).as(:document) }
+```
+
+YAML is line-oriented with significant indentation.
+
+### Mapping Rule
+
+Key-value pairs with colon separator:
+
+```ruby
+rule(:mapping) {
+  (key.as(:key) >>
+   colon >>
+   (inline_value | indented_value)).as(:mapping)
+}
+
+rule(:key) {
+  (match('[a-zA-Z_]') >> match('[a-zA-Z0-9_]').repeat).as(:key)
+}
+```
+
+Values can be inline (same line) or indented (nested block).
+
+### Inline Value
+
+Scalar on same line as key:
+
+```ruby
+rule(:inline_value) {
+  (space? >> (string | integer | float | boolean | null).as(:value) >> newline)
+}
+```
+
+Detects scalar type automatically.
+
+### Indented Value
+
+Nested block with increased indentation:
+
+```ruby
+rule(:indented_value) {
+  (newline >> indented_block.as(:block))
+}
+
+rule(:indented_block) {
+  (indent >> (mapping | list_item) >>
+   (newline >> indent >> (mapping | list_item)).repeat).as(:block)
+}
+```
+
+All items at same indent level belong to same block.
+
+### List Item Rule
+
+Hyphen-prefixed values:
+
+```ruby
+rule(:list_item) {
+  (str('-') >> space >>
+   (inline_list_value | indented_value)).as(:list_item)
+}
+```
+
+List items can contain scalars or nested structures.
+
+### String Rules
+
+Quoted and plain strings:
+
+```ruby
+rule(:quoted_string) {
+  (str('"') >>
+   (str('\\').ignore >> any | str('"').absent? >> any).repeat.as(:string) >>
+   str('"')) |
+  (str("'") >>
+   (str("'").absent? >> any).repeat.as(:string) >>
+   str("'"))
+}
+
+rule(:plain_string) {
+  (newline.absent? >> str(':').absent? >> any).repeat(1).as(:string)
+}
+```
+
+Quoted strings handle escapes; plain strings don't contain colons.
+
+### Scalar Types
+
+Type detection by pattern:
+
+```ruby
+rule(:integer) { (str('+') | str('-')).maybe >> match('[0-9]').repeat(1) }
+rule(:float) { ... match('[0-9]').repeat(1) >> str('.') >> match('[0-9]').repeat(1) }
+rule(:boolean) { str('true') | str('false') }
+rule(:null) { str('null') | str('~') }
+```
+
+Order matters: try float before integer to capture decimal point.
+
+## Output Types
+
+```ruby
+# Document with mappings
+YamlDocument.new([
+  YamlMapping.new("name", "Example"),
+  YamlMapping.new("database", { "host" => "localhost" })
+])
+
+# to_h produces:
+{
+  "name" => "Example",
+  "database" => { "host" => "localhost" }
+}
+```
+
+## Design Decisions
+
+### Why Inline vs Indented Values?
+
+YAML allows values on same line or nested. Different rules handle the distinct parsing requirements.
+
+### Why Separate String Types?
+
+Quoted strings process escape sequences; plain strings are literal. Different semantics require different rules.
+
+### Why Indentation Tracking?
+
+YAML uses indentation for structure. The parser tracks indent level to correctly nest blocks.
+
+### Why This Subset?
+
+Full YAML is complex (anchors, tags, multiline strings). This subset covers common configuration use cases.

data/ext/parsanol_native/extconf.rb

@@ -0,0 +1,4 @@
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("parsanol/parsanol_native")

data/lib/parsanol/accelerator/application.rb

@@ -0,0 +1,62 @@
+
+# @api private
+module Parsanol::Accelerator
+  class Application
+    def initialize atom, rules
+      @atom = atom
+      @rules = rules
+    end
+
+    def call
+      @atom.accept(self)
+    end
+
+    def visit_parser(root)
+      transform root.accept(self)
+    end
+    def visit_entity(name, block)
+      transform Parsanol::Atoms::Entity.new(name) { block.call.accept(self) }
+    end
+    def visit_named(name, atom)
+      transform Parsanol::Atoms::Named.new(atom.accept(self), name)
+    end
+    def visit_repetition(tag, min, max, atom)
+      transform Parsanol::Atoms::Repetition.new(atom.accept(self), min, max, tag)
+    end
+    def visit_alternative(alternatives)
+      transform Parsanol::Atoms::Alternative.new(
+        *alternatives.map { |atom| atom.accept(self) })
+    end
+    def visit_sequence(sequence)
+      transform Parsanol::Atoms::Sequence.new(
+        *sequence.map { |atom| atom.accept(self) })
+    end
+    def visit_lookahead(positive, atom)
+      transform Parsanol::Atoms::Lookahead.new(atom, positive)
+    end
+    def visit_re(regexp)
+      transform Parsanol::Atoms::Re.new(regexp)
+    end
+    def visit_str(str)
+      transform Parsanol::Atoms::Str.new(str)
+    end
+
+    def transform atom
+      @rules.each do |expr, action|
+        # Try and match each rule in turn
+        binding = Parsanol::Accelerator.match(atom, expr)
+        if binding
+          # On a successful match, allow the rule action to transform the
+          # parslet into something new. 
+          ctx = Parsanol::Context.new(binding)
+          return ctx.instance_eval(&action)
+        end
+      end # rules.each 
+
+      # If no rule matches, this is the fallback - a clean new parslet atom.
+      return atom
+    end
+  end
+end
+
+require 'parsanol/context'

data/lib/parsanol/accelerator/engine.rb

@@ -0,0 +1,112 @@
+
+require 'parsanol/atoms/visitor'
+
+module Parsanol::Accelerator
+  # @api private
+  class Apply
+    def initialize(engine, expr)
+      @engine = engine
+      @expr = expr
+    end
+
+    def visit_parser(root)
+      false
+    end
+    def visit_entity(name, block)
+      false
+    end
+    def visit_named(name, atom)
+      match(:as) do |key|
+        @engine.try_bind(key, name)
+      end
+    end
+    def visit_repetition(tag, min, max, atom)
+      match(:rep) do |e_min, e_max, expr|
+        e_min == min && e_max == max && @engine.match(atom, expr)
+      end
+    end
+    def visit_alternative(alternatives)
+      match(:alt) do |*expressions|
+        return false if alternatives.size != expressions.size
+
+        alternatives.zip(expressions).all? do |atom, expr|
+          @engine.match(atom, expr)
+        end
+      end
+    end
+    def visit_sequence(sequence)
+      match(:seq) do |*expressions|
+        return false if sequence.size != expressions.size
+
+        sequence.zip(expressions).all? do |atom, expr|
+          @engine.match(atom, expr)
+        end
+      end
+    end
+    def visit_lookahead(positive, atom)
+      match(:absent) do |expr|
+        return positive == false && @engine.match(atom, expr)
+      end
+      match(:present) do |expr|
+        return positive == true && @engine.match(atom, expr)
+      end
+    end
+    def visit_re(regexp)
+      match(:re) do |*bind_conditions|
+        bind_conditions.all? { |bind_cond| 
+          @engine.try_bind(bind_cond, regexp) }
+      end
+    end
+    def visit_str(str)
+      match(:str) do |*bind_conditions|
+        bind_conditions.all? { |bind_cond| 
+          @engine.try_bind(bind_cond, str) }
+      end
+    end
+
+    def match(type_tag)
+      expr_tag = @expr.type
+      if expr_tag == type_tag
+        yield *@expr.args
+      end
+    end
+  end
+
+  # @api private
+  class Engine
+    attr_reader :bindings
+
+    def initialize 
+      @bindings = {}
+    end
+
+    def match(atom, expr)
+      atom.accept(
+        Apply.new(self, expr))
+    end
+
+    def try_bind(variable, value)
+      if bound? variable
+        return value == lookup(variable)
+      else
+        case variable
+          when Symbol
+            bind(variable, value)
+        else
+          # This does not look like a variable - let's try matching it against
+          # the value: 
+          variable === value
+        end    
+      end
+    end
+    def bound? var
+      @bindings.has_key? var
+    end
+    def lookup var
+      @bindings[var]
+    end
+    def bind var, val
+      @bindings[var] = val
+    end
+  end
+end

data/lib/parsanol/accelerator.rb

@@ -0,0 +1,162 @@
+
+
+# Optimizes the parsers by pattern matching on the parser atoms and replacing
+# matches with better versions. See the file qed/accelerators.md for a more
+# in-depth description.
+#
+# Example: 
+#   quote = str('"')
+#   parser = quote >> (quote.absent? >> any).repeat >> quote
+#
+#   A = Accelerator # for making what follows a bit shorter
+#   optimized_parser = A.apply(parser, 
+#     A.rule( (A.str(:x).absent? >> A.any).repeat ) { GobbleUp.new(x) })
+#
+#   optimized_parser.parse('"Parsing is now fully optimized! (tm)"')
+#
+module Parsanol::Accelerator
+
+  # An expression to match against a tree of parser atoms. Normally, an
+  # expression is produced by Parsanol::Accelerator.any, 
+  # Parsanol::Accelerator.str or Parsanol::Accelerator.re.
+  #
+  # Expressions can be chained much like parslet atoms can be: 
+  #
+  #   expr.repeat(1)      # matching repetition
+  #   expr.absent?        # matching absent?
+  #   expr.present?       # matching present?
+  #   expr1 >> expr2      # matching a sequence
+  #   expr1 | expr2       # matching an alternation
+  # 
+  # @see Parsanol::Accelerator.str
+  # @see Parsanol::Accelerator.re
+  # @see Parsanol::Accelerator.any
+  #
+  # @see Parsanol::Accelerator
+  # 
+  class Expression
+    attr_reader :type
+    attr_reader :args
+
+    def initialize(type, *args)
+      @type = type
+      @args = args
+    end
+
+    # @return [Expression]
+    def >> other_expr
+      join_or_new :seq, other_expr
+    end
+
+    # @return [Expression]
+    def | other_expr
+      join_or_new :alt, other_expr
+    end
+
+    # @return [Expression]
+    def absent?
+      Expression.new(:absent, self)
+    end
+    # @return [Expression]
+    def present?
+      Expression.new(:present, self)
+    end
+
+    # @return [Expression]
+    def repeat min=0, max=nil
+      Expression.new(:rep, min, max, self)
+    end
+
+    # @return [Expression]
+    def as name
+      Expression.new(:as, name)
+    end
+
+    # @api private
+    # @return [Expression]
+    def join_or_new tag, other_expr
+      if type == tag
+        @args << other_expr
+        self
+      else
+        Expression.new(tag, self, other_expr)
+      end
+    end
+  end
+
+module_function 
+  # Returns a match expression that will match `str` parslet atoms.
+  #
+  # @return [Parsanol::Accelerator::Expression]
+  #
+  def str variable, *constraints
+    Expression.new(:str, variable, *constraints)
+  end
+
+  # Returns a match expression that will match `match` parslet atoms.
+  #
+  # @return [Parsanol::Accelerator::Expression]
+  #
+  def re variable, *constraints
+    Expression.new(:re, variable, *constraints)
+  end
+
+  # Returns a match expression that will match `any` parslet atoms.
+  #
+  # @return [Parsanol::Accelerator::Expression]
+  #
+  def any
+    Expression.new(:re, ".")
+  end
+
+  # Given a parslet atom and an expression, will determine if the expression
+  # matches the atom. If successful, returns the bindings into the pattern
+  # that were made. If no bindings had to be made to make the match successful, 
+  # the empty hash is returned. 
+  #
+  # @param atom [Parsanol::Atoms::Base] parslet atom to match against
+  # @param expr [Parsanol::Accelerator::Expression] expression to match
+  # @return [nil, Hash] bindings for the match, nil on failure
+  #
+  def match atom, expr
+    engine = Engine.new
+
+    return engine.bindings if engine.match(atom, expr)
+  end
+
+  # Constructs an accelerator rule. A rule is a matching expression and the
+  # code that should be executed once the expression could be bound to a 
+  # parser. 
+  #
+  # Example: 
+  #   Accelerator.rule(Accelerator.any) { Parslet.match('.') }
+  #
+  def rule expression, &action
+    [expression, action]
+  end
+
+  # Given a parslet atom and a set of rules, tries to match the rules 
+  # recursively through the parslet atom. Once a rule could be matched, 
+  # its action block will be called.
+  #
+  # Example: 
+  #   quote = str('"')
+  #   parser = quote >> (quote.absent? >> any).repeat >> quote
+  #
+  #   A = Accelerator # for making what follows a bit shorter
+  #   optimized_parser = A.apply(parser, 
+  #     A.rule( (A.str(:x).absent? >> A.any).repeat ) { GobbleUp.new(x) })
+  #
+  #   optimized_parser.parse('"Parsing is now fully optimized! (tm)"')
+  #
+  # @param atom [Parsanol::Atoms::Base] a parser to optimize
+  # @param *rules [Parsanol::Accelerator::Rule] rules produced by .rule
+  # @return [Parsanol::Atoms::Base] optimized parser
+  #
+  def apply atom, *rules
+    Application.new(atom, rules).call
+  end
+end
+
+require 'parsanol/accelerator/engine'
+require 'parsanol/accelerator/application'