@ohm-js/wasm 0.1.0

package/test/go/cst.go

@@ -0,0 +1,164 @@
+package main
+
+import (
+	"fmt"
+	"unsafe"
+
+	"github.com/tetratelabs/wazero/api"
+)
+
+// Code for walking a CST by hand, by accessing the raw memory.
+// Ultimately, we will want a higher-level API for this, but for now,
+// this is useful for testing/debugging.
+
+const (
+	// Node type constants
+	NodeTypeNonterminal = 0
+	NodeTypeTerminal    = -1
+	NodeTypeIter        = -2
+
+	// CST node structure size constants
+	cstNodeHeaderSize = 12 // 3 uint32 fields (count, matchLen, type)
+	uint32Size        = 4
+)
+
+// CstNode represents a node in the Concrete Syntax Tree
+type CstNode struct {
+	ruleNames []string
+	memory    api.Memory
+	base      uint32
+}
+
+// NewCstNode creates a new CstNode with the given parameters
+func NewCstNode(ruleNames []string, memory api.Memory, offset uint32) *CstNode {
+	return &CstNode{
+		ruleNames: ruleNames,
+		memory:    memory,
+		base:      offset,
+	}
+}
+
+// IsNonterminal returns true if this node represents a nonterminal
+func (n *CstNode) IsNonterminal() bool {
+	return n.Type() >= 0
+}
+
+// IsTerminal returns true if this node represents a terminal
+func (n *CstNode) IsTerminal() bool {
+	return n.Type() == NodeTypeTerminal
+}
+
+// IsIter returns true if this node represents an iteration
+func (n *CstNode) IsIter() bool {
+	return n.Type() == NodeTypeIter
+}
+
+// RuleName returns the name of the rule that created this node
+func (n *CstNode) RuleName() (string, error) {
+	data, ok := n.memory.Read(n.base+8, 4)
+	if !ok {
+		return "", fmt.Errorf("failed to read rule ID at address %d", n.base+8)
+	}
+
+	id := readInt32(data, 0)
+	if id < 0 {
+		return "", nil
+	}
+
+	if len(n.ruleNames) == 0 {
+		return fmt.Sprintf("rule_%d", id), nil
+	}
+
+	if int(id) >= len(n.ruleNames) {
+		return fmt.Sprintf("rule_%d", id), nil
+	}
+
+	if n.ruleNames[id] == "" {
+		return fmt.Sprintf("rule_%d", id), nil
+	}
+
+	return n.ruleNames[id], nil
+}
+
+// count returns the number of child nodes
+func (n *CstNode) count() (uint32, error) {
+	data, ok := n.memory.Read(n.base, 4)
+	if !ok {
+		return 0, fmt.Errorf("failed to read count at address %d", n.base)
+	}
+	return readUint32(data, 0), nil
+}
+
+// MatchLength returns the length of the matched text
+func (n *CstNode) MatchLength() (uint32, error) {
+	data, ok := n.memory.Read(n.base+4, 4)
+	if !ok {
+		return 0, fmt.Errorf("failed to read match length at address %d", n.base+4)
+	}
+	return readUint32(data, 0), nil
+}
+
+// Type returns the type of this node (0 for nonterminal, -1 for terminal, -2 for iter)
+func (n *CstNode) Type() int32 {
+	data, ok := n.memory.Read(n.base+8, 4)
+	if !ok {
+		return 0 // Return nonterminal as default in case of error
+	}
+
+	t := readInt32(data, 0)
+	if t < 0 {
+		return t
+	}
+	return NodeTypeNonterminal
+}
+
+// Children returns a slice of child nodes
+func (n *CstNode) Children() ([]*CstNode, error) {
+	count, err := n.count()
+	if err != nil {
+		return nil, err
+	}
+
+	if count == 0 {
+		return []*CstNode{}, nil
+	}
+
+	children := make([]*CstNode, count)
+
+	for i := uint32(0); i < count; i++ {
+		slotOffset := n.base + cstNodeHeaderSize + i*uint32Size
+		data, ok := n.memory.Read(slotOffset, 4)
+		if !ok {
+			return nil, fmt.Errorf("failed to read child pointer at address %d", slotOffset)
+		}
+
+		childOffset := readUint32(data, 0)
+		children[i] = NewCstNode(n.ruleNames, n.memory, childOffset)
+	}
+
+	return children, nil
+}
+
+// Helper functions for reading values from memory
+func readUint32(data []byte, offset uint32) uint32 {
+	return *(*uint32)(unsafe.Pointer(&data[offset]))
+}
+
+func readInt32(data []byte, offset uint32) int32 {
+	return *(*int32)(unsafe.Pointer(&data[offset]))
+}
+
+// GetCstRoot returns the root node of the CST from the WasmMatcher
+func GetCstRoot(matcher *WasmMatcher, ruleNames []string) (*CstNode, error) {
+	rootAddr, err := matcher.GetCstRoot()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get CST root: %v", err)
+	}
+
+	memory := matcher.GetModule().Memory()
+	if memory == nil {
+		return nil, fmt.Errorf("WebAssembly module has no memory")
+	}
+
+	return NewCstNode(ruleNames, memory, rootAddr), nil
+}

package/test/go/go.mod

@@ -0,0 +1,5 @@
+module github.com/ohmjs/ohm/packages/wasm/test/go
+
+go 1.20
+
+require github.com/tetratelabs/wazero v1.6.0

package/test/go/go.sum

@@ -0,0 +1,2 @@
+github.com/tetratelabs/wazero v1.6.0 h1:z0H1iikCdP8t+q341xqepY4EWvHEw8Es7tlqiVzlP3g=
+github.com/tetratelabs/wazero v1.6.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A=

package/test/go/matcher.go

@@ -0,0 +1,370 @@
+package main
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/tetratelabs/wazero"
+	"github.com/tetratelabs/wazero/api"
+)
+
+// Constants for memory layout
+const (
+	wasmPageSize      = 64 * 1024
+	InputBufferOffset = wasmPageSize
+	InputBufferSize   = wasmPageSize
+	MemoTableOffset   = InputBufferOffset + InputBufferSize
+)
+
+// WasmMatcher is a Go implementation of the JavaScript WasmMatcher class for Ohm
+type WasmMatcher struct {
+	runtime          wazero.Runtime
+	module           api.Module
+	input            string
+	pos              int
+	ctx              context.Context
+	ruleIds          map[string]int
+	defaultStartRule string
+	lastMatchResult  bool
+}
+
+// GetModule returns the WebAssembly module
+func (m *WasmMatcher) GetModule() api.Module {
+	return m.module
+}
+
+func NewWasmMatcher(ctx context.Context) *WasmMatcher {
+	// Create a new runtime with custom sections enabled
+	config := wazero.NewRuntimeConfig().WithCustomSections(true)
+
+	return &WasmMatcher{
+		runtime:         wazero.NewRuntimeWithConfig(ctx, config),
+		ctx:             ctx,
+		ruleIds:         make(map[string]int),
+		pos:             0,
+		lastMatchResult: false,
+	}
+}
+
+// parseRuleNames parses the rule names from the custom section data
+// The data is formatted as a WebAssembly vector of strings (each string is a length-prefixed UTF-8 bytes)
+// with LEB128-encoded lengths
+func parseRuleNames(data []byte) ([]string, error) {
+	if len(data) == 0 {
+		return nil, fmt.Errorf("empty custom section data")
+	}
+
+	// Read the number of names (vec length) as LEB128-encoded uint32
+	numNamesUint64, bytesRead := binary.Uvarint(data)
+	if bytesRead <= 0 {
+		return nil, fmt.Errorf("failed to read number of names: %v", io.ErrUnexpectedEOF)
+	}
+
+	// Ensure the value fits in uint32
+	if numNamesUint64 > uint64(^uint32(0)) {
+		return nil, fmt.Errorf("number of names exceeds maximum uint32 value")
+	}
+
+	numNames := uint32(numNamesUint64)
+	data = data[bytesRead:]
+
+	names := make([]string, numNames)
+	for i := uint32(0); i < numNames; i++ {
+		// Read the length of the name as LEB128-encoded uint32
+		nameLenUint64, bytesRead := binary.Uvarint(data)
+		if bytesRead <= 0 {
+			return nil, fmt.Errorf("failed to read name length: %v", io.ErrUnexpectedEOF)
+		}
+
+		// Ensure the value fits in uint32
+		if nameLenUint64 > uint64(^uint32(0)) {
+			return nil, fmt.Errorf("name length exceeds maximum uint32 value")
+		}
+
+		nameLen := uint32(nameLenUint64)
+		data = data[bytesRead:]
+
+		// Ensure we have enough bytes to read
+		if uint64(len(data)) < uint64(nameLen) {
+			return nil, fmt.Errorf("buffer too small to read name bytes")
+		}
+
+		// Read the name bytes
+		nameBytes := data[:nameLen]
+		data = data[nameLen:]
+
+		// Convert to string
+		names[i] = string(nameBytes)
+	}
+
+	return names, nil
+}
+
+func (m *WasmMatcher) LoadModule(wasmPath string) error {
+	// Read the WASM file
+	wasmBytes, err := os.ReadFile(wasmPath)
+	if err != nil {
+		return fmt.Errorf("error reading WASM file: %v", err)
+	}
+
+	// Create the env module with required host functions
+	_, err = m.runtime.NewHostModuleBuilder("env").
+		NewFunctionBuilder().
+		WithFunc(func(a, b, c, d int32) {
+			panic("WebAssembly module aborted execution")
+		}).
+		Export("abort").
+		NewFunctionBuilder().
+		WithFunc(func(val uint32) {
+			fmt.Printf("WASM debug: %d\n", val)
+		}).
+		Export("printI32").
+		NewFunctionBuilder().
+		WithFunc(func(ctx context.Context, mod api.Module, offset, maxLen uint32) uint32 {
+			return m.fillInputBuffer(ctx, mod, offset, maxLen)
+		}).
+		Export("fillInputBuffer").
+		Instantiate(m.ctx)
+
+	if err != nil {
+		return fmt.Errorf("failed to create host module: %v", err)
+	}
+
+	// First compile the module to access the custom sections
+	compiledModule, err := m.runtime.CompileModule(m.ctx, wasmBytes)
+	if err != nil {
+		return fmt.Errorf("error compiling module: %v", err)
+	}
+
+	// Get all custom sections from the module
+	customSections := compiledModule.CustomSections()
+	if customSections == nil {
+		return fmt.Errorf("no custom sections found in module")
+	}
+
+	var ruleNamesSection api.CustomSection
+	for _, section := range customSections {
+		if section.Name() == "ruleNames" {
+			ruleNamesSection = section
+			break
+		}
+	}
+
+	if ruleNamesSection == nil {
+		return fmt.Errorf("required custom section 'ruleNames' not found")
+	}
+
+	// Parse rule names from the custom section data
+	ruleNames, err := parseRuleNames(ruleNamesSection.Data())
+	if err != nil {
+		return fmt.Errorf("failed to parse rule names from custom section: %v", err)
+	}
+
+	// Now instantiate the module
+	m.module, err = m.runtime.InstantiateModule(m.ctx, compiledModule, wazero.NewModuleConfig())
+	if err != nil {
+		return fmt.Errorf("error instantiating module: %v", err)
+	}
+
+	// Build the ruleIds map (mapping from name to index)
+	m.ruleIds = make(map[string]int, len(ruleNames))
+	for i, name := range ruleNames {
+		m.ruleIds[name] = i
+	}
+
+	// Set the default start rule to the first rule
+	if len(ruleNames) > 0 {
+		m.defaultStartRule = ruleNames[0]
+	}
+
+	return nil
+}
+
+func (m *WasmMatcher) SetInput(input string) {
+	if m.input != input {
+		m.input = input
+		m.pos = 0
+	}
+}
+
+// SetInputFromFile reads input from a file and sets it as the current input
+func (m *WasmMatcher) SetInputFromFile(filePath string) error {
+	data, err := os.ReadFile(filePath)
+	if err != nil {
+		return fmt.Errorf("error reading input file: %v", err)
+	}
+
+	m.SetInput(string(data))
+	return nil
+}
+
+func (m *WasmMatcher) GetInput() string {
+	return m.input
+}
+
+func (m *WasmMatcher) Match() (bool, error) {
+	result, err := m.MatchRule(m.defaultStartRule)
+	if err == nil {
+		m.lastMatchResult = result
+	}
+	return result, err
+}
+
+func (m *WasmMatcher) MatchRule(ruleName string) (bool, error) {
+	m.pos = 0 // Reset position
+
+	// Get the rule ID
+	ruleId := uint64(0) // Default to 0 (start rule)
+	if ruleName != "" {
+		if id, ok := m.ruleIds[ruleName]; ok {
+			ruleId = uint64(id)
+		} else {
+			return false, fmt.Errorf("rule not found: %s", ruleName)
+		}
+	}
+
+	// Call the match function
+	matchFunc := m.module.ExportedFunction("match")
+	if matchFunc == nil {
+		return false, fmt.Errorf("match function not exported by module")
+	}
+
+	results, err := matchFunc.Call(m.ctx, ruleId)
+	if err != nil {
+		return false, fmt.Errorf("error calling match function: %v", err)
+	}
+
+	// Non-zero result means success
+	result := results[0] != 0
+	m.lastMatchResult = result
+	return result, nil
+}
+
+// GetCstRoot returns the root node address of the concrete syntax tree
+func (m *WasmMatcher) GetCstRoot() (uint32, error) {
+	getCstRootFunc := m.module.ExportedFunction("getCstRoot")
+	if getCstRootFunc == nil {
+		return 0, fmt.Errorf("getCstRoot function not exported by module")
+	}
+
+	results, err := getCstRootFunc.Call(m.ctx)
+	if err != nil {
+		return 0, fmt.Errorf("error getting CST root: %v", err)
+	}
+
+	return uint32(results[0]), nil
+}
+
+// GetCstNode returns a CstNode object for the current parse tree
+func (m *WasmMatcher) GetCstNode() (*CstNode, error) {
+	rootAddr, err := m.GetCstRoot()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get CST root: %v", err)
+	}
+
+	memory := m.module.Memory()
+	if memory == nil {
+		return nil, fmt.Errorf("WebAssembly module has no memory")
+	}
+
+	return NewCstNode(m.GetRuleNames(), memory, rootAddr), nil
+}
+
+// GetRuleNames returns the list of rule names in the grammar
+func (m *WasmMatcher) GetRuleNames() []string {
+	// If we have real rule names, use them
+	if len(m.ruleIds) > 0 {
+		// Convert the rule IDs map to a slice of rule names
+		maxID := 0
+		for _, id := range m.ruleIds {
+			if id > maxID {
+				maxID = id
+			}
+		}
+
+		ruleNames := make([]string, maxID+1)
+		for name, id := range m.ruleIds {
+			ruleNames[id] = name
+		}
+		return ruleNames
+	}
+
+	// If we don't have rule names yet, create placeholder names for ES5 grammar
+	// This is just a fallback to make tests work
+	const expectedRuleCount = 100 // More than enough for most grammars
+	ruleNames := make([]string, expectedRuleCount)
+	ruleNames[0] = "Program" // Common name for the start rule
+
+	for i := 1; i < expectedRuleCount; i++ {
+		ruleNames[i] = fmt.Sprintf("rule_%d", i)
+	}
+
+	return ruleNames
+}
+
+// fillInputBuffer is called by the WebAssembly module to get more input
+func (m *WasmMatcher) fillInputBuffer(ctx context.Context, mod api.Module, offset, maxLen uint32) uint32 {
+	// Determine how much of the input to copy
+	remaining := m.input[m.pos:]
+	bytesToWrite := len(remaining)
+	if bytesToWrite > int(maxLen) {
+		bytesToWrite = int(maxLen)
+	}
+
+	// Get module memory
+	memory := mod.Memory()
+	if memory == nil {
+		panic("WebAssembly module has no memory")
+	}
+
+	// Write to WebAssembly memory
+	if bytesToWrite > 0 {
+		// Create a buffer with our input data
+		inputData := []byte(remaining[:bytesToWrite])
+
+		// Write directly to WebAssembly memory
+		memory.Write(InputBufferOffset+offset, inputData)
+
+		// Mark end of input with 0xFF (invalid UTF-8 character)
+		memory.WriteByte(InputBufferOffset+offset+uint32(bytesToWrite), 0xFF)
+
+		// Update position
+		m.pos += bytesToWrite
+	} else {
+		// No more input, just write the terminator
+		memory.WriteByte(InputBufferOffset+offset, 0xFF)
+	}
+
+	return uint32(bytesToWrite)
+}
+
+// GetRuleId returns the ID for a rule name
+func (m *WasmMatcher) GetRuleId(ruleName string) (int, bool) {
+	id, ok := m.ruleIds[ruleName]
+	return id, ok
+}
+
+// SetDefaultStartRule sets the default start rule for matching
+func (m *WasmMatcher) SetDefaultStartRule(ruleName string) {
+	m.defaultStartRule = ruleName
+}
+
+// Close releases all resources
+func (m *WasmMatcher) Close() error {
+	if m.module != nil {
+		if err := m.module.Close(m.ctx); err != nil {
+			return err
+		}
+	}
+
+	if m.runtime != nil {
+		if err := m.runtime.Close(m.ctx); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

package/test/go/testmain.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	fmt.Println("Ohm WebAssembly Matcher - Go Implementation")
+	// Parse command line arguments
+	wasmFile := flag.String("wasm", "test/data/_add.wasm", "Path to WebAssembly file")
+	inputText := flag.String("input", "", "Input text to match against the grammar")
+	inputFile := flag.String("file", "", "Path to file containing input text to match")
+	startRule := flag.String("rule", "", "Start rule for the grammar (defaults to grammar's start rule)")
+	verbose := flag.Bool("verbose", false, "Display verbose information about CST nodes")
+	flag.Parse()
+
+	// Create a context
+	ctx := context.Background()
+
+	// Create a new WasmMatcher
+	matcher := NewWasmMatcher(ctx)
+	defer matcher.Close()
+
+	// Load the WebAssembly module
+	wasmPath := *wasmFile
+	err := matcher.LoadModule(wasmPath)
+	if err != nil {
+		fmt.Printf("Error loading module: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("Loaded WebAssembly module: %s\n", filepath.Base(wasmPath))
+
+	// Set the input text - either from direct text or from file
+	if *inputFile != "" {
+		// Read input from file
+		err = matcher.SetInputFromFile(*inputFile)
+		if err != nil {
+			fmt.Printf("Error reading input file: %v\n", err)
+			os.Exit(1)
+		}
+	} else if *inputText != "" {
+		// Set the input text directly
+		matcher.SetInput(*inputText)
+	} else {
+		fmt.Println("No input provided. Use -input flag to provide text or -file to specify an input file.")
+		os.Exit(0)
+	}
+
+	// If a start rule was specified, set it
+	if *startRule != "" {
+		fmt.Printf("Using rule: %s\n", *startRule)
+		matcher.SetDefaultStartRule(*startRule)
+	}
+
+	// Attempt to match the input
+	if *inputFile != "" {
+		fmt.Printf("Matching input file: %s\n", *inputFile)
+	} else {
+		fmt.Printf("Matching input: %q\n", matcher.GetInput())
+	}
+	success, err := matcher.Match()
+	if err != nil {
+		fmt.Printf("Error during matching: %v\n", err)
+		os.Exit(1)
+	}
+
+	if success {
+		fmt.Println("Match succeeded")
+
+		// Try to get the CST root
+		cstRoot, err := matcher.GetCstRoot()
+		if err != nil {
+			fmt.Printf("Error getting CST root: %v\n", err)
+		} else {
+			if *verbose {
+				fmt.Printf("CST root node ID: %d\n", cstRoot)
+			}
+
+			// Create a CST node from the root address
+			ruleNames := matcher.GetRuleNames()
+			node := NewCstNode(ruleNames, matcher.GetModule().Memory(), cstRoot)
+
+			nodeType := node.Type()
+			if *verbose {
+				fmt.Printf("CST Node - Type: %d\n", nodeType)
+			}
+
+			// Unparse the CST to get the original text
+			unparsedText := unparse(node, matcher.GetInput())
+			if unparsedText == matcher.GetInput() {
+				fmt.Println("Unparsed text matches input")
+			} else {
+				fmt.Println("ERROR: Unparsed text does not match input")
+				fmt.Printf("Unparsed text: %q\n", unparsedText)
+				fmt.Printf("Original input: %q\n", matcher.GetInput())
+			}
+
+			// Display verbose information about node types if requested
+			if *verbose {
+				fmt.Println("\nNode Types:")
+				fmt.Printf(" - Terminal nodes (type %d): Leaf nodes that consume input\n", NodeTypeTerminal)
+				fmt.Printf(" - Iteration nodes (type %d): Used for repetition operations\n", NodeTypeIter)
+				fmt.Printf(" - Non-terminal nodes (type %d): Internal nodes with children\n", NodeTypeNonterminal)
+			}
+		}
+	} else {
+		fmt.Println("Match failed")
+		os.Exit(1)
+	}
+}
+
+// unparse walks the CST starting from the given node and reconstructs the original text
+// It returns the reconstructed text from the terminal nodes
+func unparse(node *CstNode, input string) string {
+	var result strings.Builder
+	pos := uint32(0)
+	unparseNode(node, &pos, input, &result)
+	return result.String()
+}
+
+// unparseNode is a helper function that recursively processes nodes and builds the result
+func unparseNode(node *CstNode, pos *uint32, input string, result *strings.Builder) {
+	// Handle terminal nodes - append the consumed text to the result
+	if node.IsTerminal() {
+		matchLen, err := node.MatchLength()
+		if err != nil {
+			fmt.Printf("Error getting match length: %v\n", err)
+			return
+		}
+
+		if *pos < uint32(len(input)) && matchLen > 0 {
+			end := *pos + matchLen
+			if end > uint32(len(input)) {
+				end = uint32(len(input))
+			}
+			matchedText := input[*pos:end]
+			result.WriteString(matchedText)
+
+			// Update position only after processing terminal nodes
+			*pos += matchLen
+		}
+		return
+	}
+
+	// For all other node types (nonterminal, iteration, etc.), process children recursively
+	children, err := node.Children()
+	if err != nil {
+		fmt.Printf("Error getting children: %v\n", err)
+		return
+	}
+
+	for _, child := range children {
+		unparseNode(child, pos, input, result)
+	}
+}