parsanol 3.0.0

data/lib/parsanol/streaming_parser.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+# Parsanol::StreamingParser - Streaming Parser for Large Inputs
+#
+# Parse large inputs in chunks without loading the entire input into memory.
+# Useful for file parsing, network streams, or very large documents.
+#
+# Usage:
+#   parser = Parsanol::StreamingParser.new(json_grammar)
+#
+#   File.open("large.json") do |f|
+#     parser.parse_stream(f) do |partial_result|
+#       # Process each complete element as it's parsed
+#       process_item(partial_result)
+#     end
+#   end
+#
+# Requires native extension for full functionality.
+
+module Parsanol
+  class StreamingParser
+    # Default chunk size (4KB)
+    DEFAULT_CHUNK_SIZE = 4096
+
+    # Create a new streaming parser
+    #
+    # @param grammar [Parsanol::Parser, Parsanol::Atoms::Base] Grammar to use
+    # @param chunk_size [Integer] Size of chunks to read (default: 4096)
+    def initialize(grammar, chunk_size: DEFAULT_CHUNK_SIZE)
+      @grammar = grammar
+      @chunk_size = chunk_size
+
+      if Parsanol::Native.available?
+        grammar_json = Parsanol::Native.serialize_grammar(grammar.root)
+        @native_parser = Parsanol::Native.streaming_parser_new(grammar_json)
+      else
+        @native_parser = nil
+      end
+
+      @buffer = String.new
+      @position = 0
+    end
+
+    # Add a chunk of input
+    #
+    # @param chunk [String] Input chunk to add
+    # @return [Boolean] True if more chunks needed, false if ready for parsing
+    def add_chunk(chunk)
+      @buffer << chunk
+
+      if @native_parser
+        Parsanol::Native.streaming_parser_add_chunk(@native_parser, chunk)
+      else
+        # Pure Ruby fallback
+        false
+      end
+    end
+
+    # Parse what we have so far
+    #
+    # @return [Object, nil] Parsed result or nil if need more data
+    def parse_chunk
+      if @native_parser
+        Parsanol::Native.streaming_parser_parse_chunk(@native_parser)
+      else
+        # Pure Ruby fallback - not supported
+        raise NotImplementedError,
+          "Streaming parser requires native extension for full functionality."
+      end
+    end
+
+    # Check if we have enough data to make progress
+    #
+    # @return [Boolean] True if parser can make progress
+    def enough_data?
+      if @native_parser
+        !Parsanol::Native.streaming_parser_parse_chunk(@native_parser).nil?
+      else
+        false
+      end
+    end
+
+    # Parse entire stream (yields partial results)
+    #
+    # @param io [IO, StringIO] Input source to read from
+    # @param chunk_size [Integer] Size of chunks to read
+    # @yield [Object] Each complete element as it's parsed
+    # @return [Array] All parsed results
+    def parse_stream(io, chunk_size: @chunk_size)
+      results = []
+
+      loop do
+        chunk = io.read(chunk_size)
+        break if chunk.nil? || chunk.empty?
+
+        add_chunk(chunk)
+
+        while (result = parse_chunk)
+          results << result
+          yield result if block_given?
+        end
+      end
+
+      results
+    end
+
+    # Reset the parser for reuse
+    def reset
+      @buffer = String.new
+      @position = 0
+
+      if @native_parser
+        grammar_json = Parsanol::Native.serialize_grammar(@grammar.root)
+        @native_parser = Parsanol::Native.streaming_parser_new(grammar_json)
+      end
+    end
+
+    # Get the current buffer
+    attr_reader :buffer
+
+    # Get the chunk size
+    attr_reader :chunk_size
+  end
+end

data/lib/parsanol/string_view.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+module Parsanol
+  # Zero-copy string view that references original input.
+  #
+  # StringView avoids string copies by maintaining a reference to the
+  # original string with offset and length. Strings are only materialized
+  # when explicitly requested via #to_s.
+  #
+  # == Usage
+  #
+  #   view = StringView.new(input_string, offset: 10, length: 5)
+  #   view.to_s  # Materializes string only when needed
+  #   view[0]    # Direct character access without copying
+  #
+  # == Performance
+  #
+  # - No string allocation until to_s called
+  # - Direct character access without copying
+  # - Reduced GC pressure from intermediate strings
+  # - Caches materialized strings for reuse
+  #
+  # == Design Principles
+  #
+  # 1. Zero-Copy: Reference original string, don't copy
+  # 2. Lazy Materialization: Create strings only when to_s called
+  # 3. Caching: Cache materialized strings for reuse
+  # 4. Compatibility: Acts like String where needed
+  # 5. Extensibility: Foundation for Rope (Phase 3.2)
+  #
+  class StringView
+    # @return [String] Original input string
+    attr_reader :string
+    
+    # @return [Integer] Byte offset into string
+    attr_reader :offset
+    
+    # @return [Integer] Length in bytes
+    attr_reader :length
+
+    # Initialize a new StringView.
+    #
+    # @param string [String] Original input string
+    # @param offset [Integer] Byte offset (default: 0)
+    # @param length [Integer] Length in bytes (default: string.bytesize)
+    #
+    def initialize(string, offset: 0, length: nil)
+      @string = string
+      @offset = offset
+      @length = length || (string.bytesize - offset)
+      @materialized = nil
+    end
+
+    # Materialize to string (with caching).
+    #
+    # First call creates string slice, subsequent calls return cached.
+    # This implements lazy evaluation - strings are only created when
+    # explicitly needed, not during parsing.
+    #
+    # @return [String] Materialized string
+    #
+    def to_s
+      @materialized ||= @string.byteslice(@offset, @length)
+    end
+
+    # Get character at index (zero-copy).
+    #
+    # Direct access to character in original string without creating
+    # intermediate string objects.
+    #
+    # @param index [Integer] Zero-based index
+    # @return [String, nil] Character at index or nil
+    #
+    def [](index)
+      return nil if index < 0 || index >= @length
+      @string.byteslice(@offset + index, 1)
+    end
+
+    # Get byte size.
+    #
+    # @return [Integer] Length in bytes
+    #
+    def bytesize
+      @length
+    end
+    
+    alias size bytesize
+    alias length bytesize
+
+    # Check if empty.
+    #
+    # @return [Boolean] true if length is 0
+    #
+    def empty?
+      @length == 0
+    end
+
+    # Compare with another object.
+    #
+    # StringViews are only equal if they reference the exact same string object
+    # (by object_id) and have the same offset/length. This is consistent with
+    # the view pattern - they're views of a specific string instance.
+    #
+    # When comparing with a String, content is compared.
+    #
+    # @param other [Object] Object to compare with
+    # @return [Boolean] true if equal
+    #
+    def ==(other)
+      case other
+      when String
+        to_s == other
+      when StringView
+        # Only equal if viewing the exact same string object with same range
+        @string.equal?(other.string) &&
+          @offset == other.offset &&
+          @length == other.length
+      else
+        super
+      end
+    end
+    
+    alias eql? ==
+
+    # Hash code for hashing.
+    #
+    # Uses object_id of string to avoid materializing the view.
+    #
+    # @return [Integer] Hash code
+    #
+    def hash
+      [@string.object_id, @offset, @length].hash
+    end
+
+    # Create substring view (zero-copy).
+    #
+    # Returns a new StringView referencing a substring of this view.
+    # No string allocation occurs - just a new view with adjusted offset.
+    #
+    # @param start [Integer] Start offset (relative to view)
+    # @param len [Integer] Length
+    # @return [StringView] New view of substring
+    #
+    def slice(start, len)
+      # Handle edge cases
+      return self.class.new(@string, offset: @offset, length: 0) if len <= 0 || start >= @length
+      
+      # Clamp start to valid range [0, @length)
+      clamped_start = [[start, 0].max, @length].min
+      
+      # Calculate actual offset in original string
+      actual_offset = @offset + clamped_start
+      
+      # Calculate actual length (min of requested and available)
+      available = @length - clamped_start
+      actual_length = [len, available].min
+      
+      self.class.new(@string, offset: actual_offset, length: actual_length)
+    end
+
+    # Inspect for debugging.
+    #
+    # Shows whether string has been materialized.
+    #
+    # @return [String] Inspection string
+    #
+    def inspect
+      if @materialized
+        "#<StringView:#{object_id} @offset=#{@offset} @length=#{@length} cached=#{@materialized.inspect}>"
+      else
+        "#<StringView:#{object_id} @offset=#{@offset} @length=#{@length}>"
+      end
+    end
+
+    # Reset for pooling (if needed in future phases).
+    #
+    # Allows StringView objects to be reused from a pool.
+    #
+    # @param string [String] New string
+    # @param offset [Integer] New offset
+    # @param length [Integer] New length
+    # @return [self]
+    #
+    def reset!(string, offset, length)
+      @string = string
+      @offset = offset
+      @length = length
+      @materialized = nil
+      self
+    end
+  end
+end

data/lib/parsanol/transform.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+
+require 'parsanol/pattern'
+
+# Transforms an expression tree into something else. The transformation
+# performs a depth-first, post-order traversal of the expression tree. During
+# that traversal, each time a rule matches a node, the node is replaced by the
+# result of the block associated to the rule. Otherwise the node is accepted
+# as is into the result tree.
+#
+# This is almost what you would generally do with a tree visitor, except that
+# you can match several levels of the tree at once. 
+#
+# As a consequence of this, the resulting tree will contain pieces of the
+# original tree and new pieces. Most likely, you will want to transform the
+# original tree wholly, so this isn't a problem.
+#
+# You will not be able to create a loop, given that each node will be replaced
+# only once and then left alone. This means that the results of a replacement
+# will not be acted upon. 
+#
+# Example: 
+#
+#   class Example < Parsanol::Transform
+#     rule(:string => simple(:x)) {  # (1)
+#       StringLiteral.new(x)
+#     }
+#   end
+#
+# A tree transform (Parsanol::Transform) is defined by a set of rules. Each
+# rule can be defined by calling #rule with the pattern as argument. The block
+# given will be called every time the rule matches somewhere in the tree given
+# to #apply. It is passed a Hash containing all the variable bindings of this
+# pattern match. 
+#   
+# In the above example, (1) illustrates a simple matching rule. 
+#
+# Let's say you want to parse matching parentheses and distill a maximum nest
+# depth. You would probably write a parser like the one in example/parens.rb;
+# here's the relevant part: 
+#
+#   rule(:balanced) {
+#     str('(').as(:l) >> balanced.maybe.as(:m) >> str(')').as(:r)
+#   }
+#
+# If you now apply this to a string like '(())', you get a intermediate parse
+# tree that looks like this: 
+#
+#   {
+#     l: '(', 
+#     m: {
+#       l: '(', 
+#       m: nil, 
+#       r: ')' 
+#     }, 
+#     r: ')' 
+#   }
+#
+# This parse tree is good for debugging, but what we would really like to have
+# is just the nesting depth. This transformation rule will produce that: 
+#
+#   rule(:l => '(', :m => simple(:x), :r => ')') { 
+#     # innermost :m will contain nil
+#     x.nil? ? 1 : x+1
+#   }
+#
+# = Usage patterns
+#
+# There are four ways of using this class. The first one is very much
+# recommended, followed by the second one for generality. The other ones are
+# omitted here. 
+#
+# Recommended usage is as follows: 
+#
+#   class MyTransformator < Parsanol::Transform
+#     rule(...) { ... }
+#     rule(...) { ... }
+#     # ...
+#   end
+#   MyTransformator.new.apply(tree)
+#
+# Alternatively, you can use the Transform class as follows: 
+#
+#   transform = Parsanol::Transform.new do
+#     rule(...) { ... }
+#   end
+#   transform.apply(tree)
+#
+# = Execution context
+#
+# The execution context of action blocks differs depending on the arity of 
+# said blocks. This can be confusing. It is however somewhat intentional. You 
+# should not create fat Transform descendants containing a lot of helper methods, 
+# instead keep your AST class construction in global scope or make it available
+# through a factory. The following piece of code illustrates usage of global
+# scope: 
+#
+#   transform = Parsanol::Transform.new do
+#     rule(...) { AstNode.new(a_variable) }
+#     rule(...) { Ast.node(a_variable) } # modules are nice
+#   end
+#   transform.apply(tree)
+#
+# And here's how you would use a class builder (a factory):
+#
+#   transform = Parsanol::Transform.new do
+#     rule(...) { builder.add_node(a_variable) }
+#     rule(...) { |d| d[:builder].add_node(d[:a_variable]) }
+#   end
+#   transform.apply(tree, :builder => Builder.new)
+#
+# As you can see, Transform allows you to inject local context for your rule
+# action blocks to use. 
+#
+class Parsanol::Transform
+  # FIXME: Maybe only part of it? Or maybe only include into constructor
+  # context?
+  include Parsanol
+
+  class << self
+    # FIXME: Only do this for subclasses?
+    include Parsanol
+    
+    # Define a rule for the transform subclass. 
+    #
+    def rule(expression, &block)
+      @__transform_rules ||= []
+      # Prepend new rules so they have higher precedence than older rules
+      @__transform_rules.unshift([Parsanol::Pattern.new(expression), block])
+    end
+    
+    # Allows accessing the class' rules
+    #
+    def rules 
+      @__transform_rules ||= []
+    end
+
+    def inherited(subclass)
+      super
+      subclass.instance_variable_set(:@__transform_rules, rules.dup)
+    end
+  end
+  
+  def initialize(raise_on_unmatch=false, &block) 
+    @raise_on_unmatch = raise_on_unmatch
+    @rules = []
+    
+    if block
+      instance_eval(&block)
+    end
+  end
+  
+  # Defines a rule to be applied whenever apply is called on a tree. A rule
+  # is composed of two parts: 
+  # 
+  # * an *expression pattern*
+  # * a *transformation block*
+  #
+  def rule(expression, &block)
+    # Prepend new rules so they have higher precedence than older rules
+    @rules.unshift([Parsanol::Pattern.new(expression), block])
+  end
+  
+  # Applies the transformation to a tree that is generated by Parsanol::Parser
+  # or a simple parslet. Transformation will proceed down the tree, replacing
+  # parts/all of it with new objects. The resulting object will be returned. 
+  #
+  # Using the context parameter, you can inject bindings for the transformation.
+  # This can be used to allow access to the outside world from transform blocks,
+  # like so:
+  # 
+  #   document = # some class that you act on
+  #   transform.apply(tree, document: document)
+  #   
+  # The above will make document available to all your action blocks: 
+  #
+  #   # Variant A
+  #   rule(...) { document.foo(bar) }
+  #   # Variant B
+  #   rule(...) { |d| d[:document].foo(d[:bar]) }
+  #
+  # @param obj PORO ast to transform
+  # @param context start context to inject into the bindings. 
+  #
+  def apply(obj, context=nil)
+    transform_elt(
+      case obj
+        when Hash
+          recurse_hash(obj, context)
+        when Array
+          recurse_array(obj, context)
+      else
+        obj
+      end, 
+      context
+    )
+  end
+  
+  # Executes the block on the bindings obtained by Pattern#match, if such a match
+  # can be made. Depending on the arity of the given block, it is called in 
+  # one of two environments: the current one or a clean toplevel environment.
+  #
+  # If you would like the current environment preserved, please use the 
+  # arity 1 variant of the block. Alternatively, you can inject a context object
+  # and call methods on it (think :ctx => self).
+  #
+  #   # the local variable a is simulated
+  #   t.call_on_match(:a => :b) { a } 
+  #   # no change of environment here
+  #   t.call_on_match(:a => :b) { |d| d[:a] }
+  #
+  def call_on_match(bindings, block)
+    if block
+      if block.arity == 1
+        return block.call(bindings)
+      else
+        context = Context.new(bindings)
+        return context.instance_eval(&block)
+      end
+    end
+  end
+  
+  # Allow easy access to all rules, the ones defined in the instance and the 
+  # ones predefined in a subclass definition. 
+  #
+  def rules 
+    self.class.rules + @rules
+  end
+  
+  # @api private 
+  #
+  def transform_elt(elt, context) 
+    rules.each do |pattern, block|
+      if bindings=pattern.match(elt, context)
+        # Produces transformed value
+        return call_on_match(bindings, block)
+      end
+    end
+    
+    # No rule matched - element is not transformed
+    if @raise_on_unmatch && elt.is_a?(Hash)
+      elt_types = elt.map do |key, value|
+        [ key, value.class ]
+      end.to_h
+      raise NotImplementedError, "Failed to match `#{elt_types.inspect}`"
+    else
+      return elt
+    end
+  end
+
+  # @api private 
+  #
+  def recurse_hash(hsh, ctx) 
+    hsh.inject({}) do |new_hsh, (k,v)|
+      new_hsh[k] = apply(v, ctx)
+      new_hsh
+    end
+  end
+  # @api private 
+  #
+  def recurse_array(ary, ctx) 
+    ary.map { |elt| apply(elt, ctx) }
+  end
+end
+
+require 'parsanol/context'

data/lib/parsanol/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Parsanol
+  VERSION = '3.0.0'
+end

data/lib/parsanol/wasm/README.md

@@ -0,0 +1,80 @@
+# @parsanol/wasm
+
+High-performance PEG parser using WebAssembly. Designed for use with Opal (Ruby in JavaScript) and general JavaScript applications.
+
+## Features
+
+- **18-44x faster** than pure Ruby parser
+- **99.5% fewer allocations**
+- Works in browsers and Node.js
+- Full TypeScript support
+- Compatible with Opal
+
+## Installation
+
+```bash
+npm install @parsanol/wasm
+```
+
+## Usage
+
+### Browser/ESM
+
+```html
+<script type="module">
+  import { initParsanol, ParsanolParser } from '@parsanol/wasm';
+
+  // Initialize WASM (call once)
+  await initParsanol();
+
+  // Create parser from grammar JSON
+  const grammar = {
+    atoms: [
+      { Str: { pattern: "hello" } }
+    ],
+    root: 0
+  };
+
+  const parser = new ParsanolParser(grammar);
+
+  // Parse input
+  const result = parser.parse('hello');
+  console.log(result); // "hello"
+</script>
+```
+
+### Node.js
+
+```javascript
+const { initParsanol, ParsanolParser } = require('@parsanol/wasm');
+
+async function main() {
+  await initParsanol();
+
+  const parser = new ParsanolParser(grammarJson);
+  const result = parser.parse('input text');
+  console.log(result);
+}
+
+main();
+```
+
+### Opal (Ruby in Browser)
+
+```ruby
+# First initialize WASM in JavaScript:
+# Parsanol::WasmParser.init.then { puts "ready" }
+
+require 'parsanol/wasm_parser'
+
+grammar_json = {
+  atoms: [
+    { Str: { pattern: "hello" } }
+  ],
+  root: 0
+}.to_json
+
+parser = Parsanol::WasmParser.new(grammar_json)
+result = parser.parse('hello')
+puts result  # => "hello"
+```