@atomic-ehr/fhirpath 0.0.1-canary.0c6931e.20250727185306

package/src/parser/token-navigator.ts

@@ -0,0 +1,110 @@
+import type { Token } from '../lexer/token';
+import { TokenType } from '../lexer/token';
+
+/**
+ * Handles token navigation and lookahead operations for the parser
+ */
+export class TokenNavigator {
+  private tokens: Token[];
+  private current: number = 0;
+
+  constructor(tokens: Token[]) {
+    this.tokens = tokens;
+  }
+
+  /**
+   * Check if the current token matches any of the given types
+   */
+  match(...types: TokenType[]): boolean {
+    for (const type of types) {
+      if (this.check(type)) {
+        this.advance();
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Check if the current token is of the given type
+   */
+  check(type: TokenType): boolean {
+    if (this.isAtEnd()) return false;
+    return this.peek().type === type;
+  }
+
+  /**
+   * Advance to the next token and return the previous one
+   */
+  advance(): Token {
+    if (!this.isAtEnd()) this.current++;
+    return this.previous();
+  }
+
+  /**
+   * Check if we've reached the end of tokens
+   */
+  isAtEnd(): boolean {
+    return this.peek().type === TokenType.EOF;
+  }
+
+  /**
+   * Get the current token without advancing
+   */
+  peek(): Token {
+    if (this.tokens.length === 0) {
+      // Return a synthetic EOF token when there are no tokens
+      return {
+        type: TokenType.EOF,
+        value: '',
+        position: { line: 1, column: 1, offset: 0 }
+      } as Token;
+    }
+    return this.tokens[this.current] ?? this.tokens[this.tokens.length - 1]!;
+  }
+
+  /**
+   * Get the previous token
+   */
+  previous(): Token {
+    return this.tokens[this.current - 1]!;
+  }
+
+  /**
+   * Get current position in token stream
+   */
+  getCurrentPosition(): number {
+    return this.current;
+  }
+
+  /**
+   * Restore position in token stream
+   */
+  restorePosition(position: number): void {
+    this.current = position;
+  }
+
+  /**
+   * Consume a token of the expected type or throw an error
+   */
+  consume(type: TokenType, errorCallback: (token: Token) => Error): Token {
+    if (this.check(type)) return this.advance();
+    throw errorCallback(this.peek());
+  }
+
+  /**
+   * Skip tokens while a condition is true
+   */
+  skipWhile(condition: (token: Token) => boolean): void {
+    while (!this.isAtEnd() && condition(this.peek())) {
+      this.advance();
+    }
+  }
+
+  /**
+   * Check if any of the token types match current token
+   */
+  checkAny(types: TokenType[]): boolean {
+    return types.some(type => this.check(type));
+  }
+}

package/src/parser/types.ts

@@ -0,0 +1,60 @@
+import type { ASTNode } from './ast';
+import type { ErrorCode } from '../api/errors';
+
+export interface ParserOptions {
+  maxErrors?: number;
+  throwOnError?: boolean;      // When true, throws on first error instead of collecting diagnostics
+  trackRanges?: boolean;        // Enable source range tracking for each AST node (useful for IDEs)
+  errorRecovery?: boolean;      // Enable error recovery to continue parsing after errors (useful for IDEs)
+}
+
+export interface ParseResult {
+  ast: ASTNode;
+  diagnostics: ParseDiagnostic[];
+  hasErrors: boolean;
+  isPartial?: boolean;                    // Present when errorRecovery is enabled
+  ranges?: Map<ASTNode, TextRange>;       // Present when trackRanges is enabled
+}
+
+export interface ParseDiagnostic {
+  range: TextRange;
+  severity: DiagnosticSeverity;
+  code: ErrorCode;
+  message: string;
+  source: 'fhirpath-parser';
+  relatedInformation?: RelatedInformation[];
+}
+
+export enum DiagnosticSeverity {
+  Error = 1,
+  Warning = 2,
+  Information = 3,
+  Hint = 4
+}
+
+export interface TextRange {
+  start: Position;
+  end: Position;
+}
+
+export interface Position {
+  line: number;
+  character: number;
+  offset: number;
+}
+
+export interface RelatedInformation {
+  location: TextRange;
+  message: string;
+}
+
+export enum ParseContext {
+  Expression,
+  FunctionCall,
+  IndexExpression,
+  BinaryExpression,
+  UnaryExpression,
+  CollectionLiteral,
+  TypeCast,
+  MembershipTest
+}

package/src/parser2/index.md

@@ -0,0 +1,177 @@
+# FHIRPath Parser2 - Recursive Descent with Pratt Parsing
+
+This parser implements a recursive-descent parser with Pratt operator precedence parsing for FHIRPath expressions. It's designed to be simple, efficient, and self-contained.
+
+## Architecture Overview
+
+The parser consists of:
+- **Lexical Analysis**: Uses `lexer2` to tokenize the input
+- **Recursive Descent**: Top-down parsing for primary expressions
+- **Pratt Parsing**: Handles operator precedence and associativity
+- **AST Construction**: Builds a typed Abstract Syntax Tree
+
+## Parser Flow
+
+### 1. Initialization
+
+```typescript
+const parser = new Parser(input);
+// 1. Creates a Lexer instance
+// 2. Tokenizes entire input upfront
+// 3. Stores tokens in array for random access
+```
+
+### 2. Main Entry Point
+
+```typescript
+parse(): ASTNode
+```
+- Calls `expression()` to parse the entire input
+- Ensures all tokens are consumed (no trailing tokens)
+- Returns the root AST node
+
+### 3. Expression Parsing (Pratt Algorithm)
+
+```typescript
+parseExpressionWithPrecedence(minPrecedence): ASTNode
+```
+
+The core of the parser uses Pratt parsing:
+
+1. **Parse Primary Expression** - Get the left-hand side
+2. **Parse Operators Loop** - While we have operators with precedence >= minPrecedence:
+   - Consume the operator
+   - Parse right-hand side with appropriate precedence
+   - Create binary/special nodes
+
+#### Operator Precedence (highest to lowest):
+
+| Precedence | Operators | Associativity |
+|------------|-----------|---------------|
+| 100 | `.` (member), `[` (index), `(` (call) | Left |
+| 90 | `is`, `as` | Left |
+| 80 | `*`, `/`, `div`, `mod` | Left |
+| 70 | `+`, `-` | Left |
+| 60 | `&` | Left |
+| 50 | `<`, `>`, `<=`, `>=` | Left |
+| 40 | `=`, `!=`, `~`, `!~` | Left |
+| 35 | `in`, `contains` | Left |
+| 30 | `and` | Left |
+| 20 | `or`, `xor` | Left |
+| 10 | `implies` | Right |
+| 5 | `|` (union) | Left |
+
+### 4. Primary Expression Parsing
+
+```typescript
+parsePrimary(): ASTNode
+```
+
+Handles atomic expressions:
+- **Literals**: Numbers, strings, booleans, null, datetime, time
+- **Variables**: `$this`, `$index`, `$total`, `%env`
+- **Identifiers**: Simple or delimited (backtick) identifiers
+- **Parentheses**: `(expression)`
+- **Collections**: `{element1, element2, ...}`
+- **Unary Operators**: `+expr`, `-expr`
+
+### 5. Special Constructs
+
+#### Member Access and Function Calls
+
+After parsing `.identifier`, the parser checks if it's followed by `(` to distinguish between:
+- Property access: `Patient.name`
+- Method call: `Patient.name.where(...)`
+
+```typescript
+parseInvocation(): ASTNode
+// 1. Parse identifier after dot
+// 2. Check for '(' to determine if it's a function call
+// 3. Return appropriate node type
+```
+
+#### Type Operations
+
+- **Type Test**: `expression is TypeName`
+- **Type Cast**: `expression as TypeName`
+
+Both create special node types rather than generic binary operators.
+
+#### Union Operator
+
+The `|` operator is special - it creates/extends a UnionNode with multiple operands:
+```typescript
+a | b | c  =>  UnionNode { operands: [a, b, c] }
+```
+
+### 6. AST Node Types
+
+All AST nodes implement the base `ASTNode` interface:
+```typescript
+interface ASTNode {
+  type: NodeType;
+  position: Position;
+}
+```
+
+Node types include:
+- **Identifier**: Simple identifiers like `name`
+- **TypeOrIdentifier**: Uppercase identifiers like `Patient`
+- **Literal**: Values with type information
+- **Binary**: Binary operators with left/right operands
+- **Unary**: Unary operators with single operand
+- **Function**: Function calls with name and arguments
+- **Variable**: Special variables (`$this`, `%env`)
+- **Index**: Array/collection indexing
+- **Union**: Multiple operands joined by `|`
+- **MembershipTest**: `is` operator
+- **TypeCast**: `as` operator
+- **Collection**: `{}` expressions
+
+## Example Parse Flow
+
+For expression: `Patient.name.where(use = 'official').given`
+
+1. **Primary**: Parse `Patient` → TypeOrIdentifierNode
+2. **Dot operator** (precedence 100):
+   - Parse `name` → IdentifierNode
+   - Create BinaryNode(DOT, Patient, name)
+3. **Dot operator**:
+   - Parse `where` → IdentifierNode
+   - Check for `(` → It's a function call!
+   - Parse arguments: `use = 'official'`
+   - Create FunctionNode(where, [BinaryNode(EQ, use, 'official')])
+   - Create BinaryNode(DOT, Patient.name, where(...))
+4. **Dot operator**:
+   - Parse `given` → IdentifierNode
+   - Create BinaryNode(DOT, Patient.name.where(...), given)
+
+## Key Design Decisions
+
+1. **Tokenize Upfront**: All tokens are generated before parsing begins, allowing lookahead and backtracking if needed.
+
+2. **Pratt Parsing**: Eliminates the need for separate grammar rules for each precedence level, making the parser more maintainable.
+
+3. **Special Node Types**: Instead of generic nodes, specific types like `MembershipTest` and `TypeCast` preserve semantic information.
+
+4. **Self-Contained**: All types are defined within the module, making it independent and easy to understand.
+
+5. **Position Tracking**: Every node includes position information for error reporting and tooling support.
+
+## Usage
+
+```typescript
+import { parse } from './parser2';
+
+const ast = parse('Patient.name.given');
+console.log(JSON.stringify(ast, null, 2));
+```
+
+## Error Handling
+
+The parser throws descriptive errors for:
+- Unexpected tokens
+- Missing closing delimiters
+- Invalid syntax constructs
+
+Errors include the problematic token value for debugging.

package/src/parser2/index.perf.test.ts

@@ -0,0 +1,184 @@
+import { describe, it } from 'bun:test';
+import { Parser } from './index';
+import * as fs from 'fs';
+import * as path from 'path';
+
+describe('Parser2 Performance', () => {
+  it('measures parser performance on fixture expressions', () => {
+    runPerformanceTest();
+  });
+});
+
+function runPerformanceTest() {
+    const fixturesPath = path.join(process.cwd(), 'test', 'fixtures');
+    const iterations = 5000; // Fewer iterations than lexer since parsing is more expensive
+    
+    // Read all fixture files
+    const fixtureFiles = fs.readdirSync(fixturesPath)
+      .filter(file => file.endsWith('.json'))
+      .map(file => ({
+        name: file,
+        path: path.join(fixturesPath, file)
+      }));
+    
+    console.log(`\nRunning parser2 performance tests with ${iterations} iterations per expression\n`);
+    
+    let totalExpressions = 0;
+    let totalIterations = 0;
+    let totalTime = 0;
+    let totalTokens = 0;
+    let totalNodes = 0;
+    const expressionStats: { expression: string; time: number; tokens: number; nodes: number }[] = [];
+    
+    for (const fixture of fixtureFiles) {
+      console.log(`Processing ${fixture.name}...`);
+      
+      const content = fs.readFileSync(fixture.path, 'utf-8');
+      const expressions: string[] = JSON.parse(content);
+      
+      for (const expression of expressions) {
+        if (!expression) continue;
+        
+        // Warm up run and get stats
+        const warmupParser = new Parser(expression);
+        const ast = warmupParser.parse();
+        const tokenCount = countTokens(warmupParser);
+        const nodeCount = countNodes(ast);
+        
+        // Measure total time for all iterations
+        const start = performance.now();
+        for (let j = 0; j < iterations; j++) {
+          const parser = new Parser(expression);
+          parser.parse();
+        }
+        const end = performance.now();
+        
+        const totalTimeForExpression = end - start;
+        totalTime += totalTimeForExpression;
+        totalExpressions++;
+        totalIterations += iterations;
+        totalTokens += tokenCount * iterations;
+        totalNodes += nodeCount * iterations;
+        
+        expressionStats.push({
+          expression,
+          time: totalTimeForExpression / iterations,
+          tokens: tokenCount,
+          nodes: nodeCount
+        });
+      }
+    }
+    
+    const avgTimePerExpression = totalTime / totalIterations;
+    const avgTokensPerExpression = totalTokens / totalIterations;
+    const avgNodesPerExpression = totalNodes / totalIterations;
+    
+    // Sort by time to find slowest expressions
+    expressionStats.sort((a, b) => b.time - a.time);
+    
+    console.log('\n' + '='.repeat(70));
+    console.log('PARSER2 PERFORMANCE RESULTS');
+    console.log('='.repeat(70));
+    console.log(`Total expressions: ${totalExpressions}`);
+    console.log(`Total iterations: ${totalIterations}`);
+    console.log(`Total time: ${(totalTime / 1000).toFixed(2)}s`);
+    console.log(`Time per expression: ${avgTimePerExpression.toFixed(4)}ms`);
+    console.log(`Expressions per second: ${(1000 / avgTimePerExpression).toFixed(0)}`);
+    console.log(`Average tokens per expression: ${avgTokensPerExpression.toFixed(1)}`);
+    console.log(`Average AST nodes per expression: ${avgNodesPerExpression.toFixed(1)}`);
+    
+    console.log('\n' + '='.repeat(70));
+    console.log('TOP 10 SLOWEST EXPRESSIONS');
+    console.log('='.repeat(70));
+    console.log('Time (ms) | Tokens | Nodes | Expression');
+    console.log('-'.repeat(70));
+    
+    for (let i = 0; i < Math.min(10, expressionStats.length); i++) {
+      const stat = expressionStats[i];
+      const expr = stat.expression.length > 40 
+        ? stat.expression.substring(0, 37) + '...' 
+        : stat.expression;
+      console.log(
+        `${stat.time.toFixed(4).padStart(9)} | ` +
+        `${stat.tokens.toString().padStart(6)} | ` +
+        `${stat.nodes.toString().padStart(5)} | ` +
+        expr
+      );
+    }
+    
+    // Calculate complexity metrics
+    const complexityStats = expressionStats.map(stat => ({
+      ...stat,
+      timePerToken: stat.time / stat.tokens,
+      timePerNode: stat.time / stat.nodes
+    }));
+    
+    console.log('\n' + '='.repeat(70));
+    console.log('COMPLEXITY ANALYSIS');
+    console.log('='.repeat(70));
+    
+    // Group by token count ranges
+    const tokenRanges = [
+      { min: 0, max: 5, label: '1-5 tokens' },
+      { min: 5, max: 10, label: '6-10 tokens' },
+      { min: 10, max: 20, label: '11-20 tokens' },
+      { min: 20, max: 50, label: '21-50 tokens' },
+      { min: 50, max: Infinity, label: '50+ tokens' }
+    ];
+    
+    console.log('\nPerformance by expression complexity:');
+    console.log('Token Range  | Count | Avg Time (ms) | Time/Token (μs)');
+    console.log('-'.repeat(55));
+    
+    for (const range of tokenRanges) {
+      const inRange = complexityStats.filter(
+        s => s.tokens > range.min && s.tokens <= range.max
+      );
+      if (inRange.length > 0) {
+        const avgTime = inRange.reduce((sum, s) => sum + s.time, 0) / inRange.length;
+        const avgTimePerToken = inRange.reduce((sum, s) => sum + s.timePerToken, 0) / inRange.length;
+        console.log(
+          `${range.label.padEnd(12)} | ${inRange.length.toString().padStart(5)} | ` +
+          `${avgTime.toFixed(4).padStart(13)} | ${(avgTimePerToken * 1000).toFixed(2).padStart(15)}`
+        );
+      }
+    }
+}
+
+function countTokens(parser: Parser): number {
+  // Access the private tokens array via reflection
+  return (parser as any).tokens.length;
+}
+
+function countNodes(node: any): number {
+  if (!node) return 0;
+  
+  let count = 1;
+  
+  // Count nodes in common structures
+  if (node.left) count += countNodes(node.left);
+  if (node.right) count += countNodes(node.right);
+  if (node.operand) count += countNodes(node.operand);
+  if (node.expression) count += countNodes(node.expression);
+  if (node.index) count += countNodes(node.index);
+  if (node.name && typeof node.name === 'object') count += countNodes(node.name);
+  
+  // Count nodes in arrays
+  if (node.arguments) {
+    for (const arg of node.arguments) {
+      count += countNodes(arg);
+    }
+  }
+  if (node.elements) {
+    for (const elem of node.elements) {
+      count += countNodes(elem);
+    }
+  }
+  if (node.operands) {
+    for (const op of node.operands) {
+      count += countNodes(op);
+    }
+  }
+  
+  return count;
+}