@atomic-ehr/fhirpath 0.0.2 → 0.0.3

package/src/lexer.ts

@@ -23,6 +23,7 @@ export enum TokenType {
   DATETIME = 4,
   TIME = 5,
   QUANTITY = 6,     // Quantity literals like 5 'mg'
+  DATE = 7,         // Date literals like @2020-01-01
   
   // Operators (all symbol operators consolidated)
   OPERATOR = 10,    // +, -, *, /, <, >, <=, >=, =, !=, ~, !~, |, &
@@ -76,8 +77,6 @@ export class Lexer {
   private position: number = 0;
   private line: number = 1;     // Legacy: 1-based for backward compatibility
   private column: number = 1;   // Legacy: 1-based for backward compatibility
-  private lspLine: number = 0;     // LSP: zero-based
-  private lspCharacter: number = 0; // LSP: zero-based character within line
   private options: LexerOptions;
   private lineOffsets: number[] = [0]; // Start positions of each line
   
@@ -160,25 +159,13 @@ export class Lexer {
     const start = this.position;
     const startLine = this.line;
     const startColumn = this.column;
-    const char = this.input[this.position];
+    const char = this.input[this.position]!;
     const charCode = this.input.charCodeAt(this.position);
     
     // Single character tokens
     switch (char) {
-      case '+':
-        this.advance();
-        return this.createToken(TokenType.OPERATOR, '+', start, this.position, startLine, startColumn);
-        
-      case '-':
-        this.advance();
-        return this.createToken(TokenType.OPERATOR, '-', start, this.position, startLine, startColumn);
-        
-      case '*':
-        this.advance();
-        return this.createToken(TokenType.OPERATOR, '*', start, this.position, startLine, startColumn);
-        
       case '/':
-        // Check for comments
+        // Handle comments first; otherwise treat as operator
         if (this.peek() === '/') {
           if (this.options.preserveTrivia) {
             const commentStart = this.position;
@@ -217,53 +204,9 @@ export class Lexer {
             return null;
           }
         }
-        this.advance();
-        return this.createToken(TokenType.OPERATOR, '/', start, this.position, startLine, startColumn);
-        
-      case '<':
-        this.advance();
-        if (this.current() === '=') {
-          this.advance();
-          return this.createToken(TokenType.OPERATOR, '<=', start, this.position, startLine, startColumn);
-        }
-        return this.createToken(TokenType.OPERATOR, '<', start, this.position, startLine, startColumn);
-        
-      case '>':
-        this.advance();
-        if (this.current() === '=') {
-          this.advance();
-          return this.createToken(TokenType.OPERATOR, '>=', start, this.position, startLine, startColumn);
-        }
-        return this.createToken(TokenType.OPERATOR, '>', start, this.position, startLine, startColumn);
-        
-      case '=':
-        this.advance();
-        return this.createToken(TokenType.OPERATOR, '=', start, this.position, startLine, startColumn);
-        
-      case '!':
-        this.advance();
-        if (this.current() === '=') {
-          this.advance();
-          return this.createToken(TokenType.OPERATOR, '!=', start, this.position, startLine, startColumn);
-        } else if (this.current() === '~') {
-          this.advance();
-          return this.createToken(TokenType.OPERATOR, '!~', start, this.position, startLine, startColumn);
-        }
-        // '!' alone is not a valid token in FHIRPath
-        throw this.error(`Unexpected character '!' at position ${start}`);
-        
-      case '~':
-        this.advance();
-        return this.createToken(TokenType.OPERATOR, '~', start, this.position, startLine, startColumn);
-        
-      case '|':
-        this.advance();
-        return this.createToken(TokenType.OPERATOR, '|', start, this.position, startLine, startColumn);
-        
-      case '&':
-        this.advance();
-        return this.createToken(TokenType.OPERATOR, '&', start, this.position, startLine, startColumn);
-        
+        // Fall through to operator matching after comment checks
+        return this.readOperator(start, startLine, startColumn);
+
       case '.':
         this.advance();
         return this.createToken(TokenType.DOT, '.', start, this.position, startLine, startColumn);
@@ -315,7 +258,12 @@ export class Lexer {
       case '$':
         return this.readSpecialIdentifier();
     }
-    
+
+    // Greedy operator matching for other operator starters
+    if (this.isOperatorStarter(char)) {
+      return this.readOperator(start, startLine, startColumn);
+    }
+
     // Numbers
     if (charCode >= 48 && charCode <= 57) { // 0-9
       return this.readNumber();
@@ -340,17 +288,7 @@ export class Lexer {
     this.advance();
     
     // Continue with alphanumeric or underscore
-    while (this.position < this.input.length) {
-      const charCode = this.input.charCodeAt(this.position);
-      if ((charCode >= 65 && charCode <= 90) || // A-Z
-          (charCode >= 97 && charCode <= 122) || // a-z
-          (charCode >= 48 && charCode <= 57) || // 0-9
-          charCode === 95) { // _
-        this.advance();
-      } else {
-        break;
-      }
-    }
+    this.scanIdentifierBody();
     
     const value = this.input.substring(start, this.position);
     return this.createToken(TokenType.IDENTIFIER, value, start, this.position, startLine, startColumn);
@@ -361,28 +299,10 @@ export class Lexer {
     const startLine = this.line;
     const startColumn = this.column;
     
-    this.advance(); // Skip opening `
-    
-    while (this.position < this.input.length) {
-      const char = this.current();
-      
-      if (char === '`') {
-        this.advance(); // Skip closing `
-        const value = this.input.substring(start, this.position);
-        return this.createToken(TokenType.IDENTIFIER, value, start, this.position, startLine, startColumn);
-      }
-      
-      if (char === '\\') {
-        this.advance(); // Skip escape character
-        if (this.position >= this.input.length) {
-          throw this.error('Unterminated delimited identifier');
-        }
-      }
-      
-      this.advance();
-    }
-    
-    throw this.error('Unterminated delimited identifier');
+    // Current must be opening backtick
+    this.scanQuoted('`', 'Unterminated delimited identifier');
+    const value = this.input.substring(start, this.position);
+    return this.createToken(TokenType.IDENTIFIER, value, start, this.position, startLine, startColumn);
   }
   
   private readSpecialIdentifier(): Token {
@@ -392,18 +312,8 @@ export class Lexer {
     
     this.advance(); // Skip $
     
-    // Read the identifier part
-    while (this.position < this.input.length) {
-      const charCode = this.input.charCodeAt(this.position);
-      if ((charCode >= 65 && charCode <= 90) || // A-Z
-          (charCode >= 97 && charCode <= 122) || // a-z
-          (charCode >= 48 && charCode <= 57) || // 0-9
-          charCode === 95) { // _
-        this.advance();
-      } else {
-        break;
-      }
-    }
+    // Read the identifier part (may be empty)
+    this.scanIdentifierBody();
     
     const value = this.input.substring(start, this.position);
     
@@ -423,108 +333,36 @@ export class Lexer {
     
     if (char === '`') {
       // Delimited identifier: %`identifier`
-      this.advance(); // Skip opening `
-      
-      while (this.position < this.input.length) {
-        const ch = this.current();
-        
-        if (ch === '`') {
-          this.advance(); // Skip closing `
-          const value = this.input.substring(start, this.position);
-          return this.createToken(TokenType.ENVIRONMENT_VARIABLE, value, start, this.position, startLine, startColumn);
-        }
-        
-        if (ch === '\\') {
-          this.advance(); // Skip escape character
-          if (this.position >= this.input.length) {
-            throw this.error('Unterminated environment variable');
-          }
-        }
-        
-        this.advance();
-      }
-      
-      throw this.error('Unterminated environment variable');
+      this.scanQuoted('`', 'Unterminated environment variable');
+      const value = this.input.substring(start, this.position);
+      return this.createToken(TokenType.ENVIRONMENT_VARIABLE, value, start, this.position, startLine, startColumn);
     } else if (char === "'") {
       // String format (backwards compatibility): %'identifier'
-      this.advance(); // Skip opening '
-      
-      while (this.position < this.input.length) {
-        const ch = this.current();
-        
-        if (ch === "'") {
-          this.advance(); // Skip closing '
-          const value = this.input.substring(start, this.position);
-          return this.createToken(TokenType.ENVIRONMENT_VARIABLE, value, start, this.position, startLine, startColumn);
-        }
-        
-        if (ch === '\\') {
-          this.advance(); // Skip escape character
-          if (this.position >= this.input.length) {
-            throw this.error('Unterminated environment variable');
-          }
-        }
-        
-        this.advance();
-      }
-      
-      throw this.error('Unterminated environment variable');
+      this.scanQuoted("'", 'Unterminated environment variable');
+      const value = this.input.substring(start, this.position);
+      return this.createToken(TokenType.ENVIRONMENT_VARIABLE, value, start, this.position, startLine, startColumn);
     } else {
       // Simple identifier: %identifier
       const charCode = this.input.charCodeAt(this.position);
-      if (!((charCode >= 65 && charCode <= 90) || // A-Z
-            (charCode >= 97 && charCode <= 122) || // a-z
-            charCode === 95)) { // _
+      if (!this.isIdentifierHead(charCode)) {
         throw this.error('Invalid environment variable name');
       }
-      
-      // Read the identifier part
-      while (this.position < this.input.length) {
-        const charCode = this.input.charCodeAt(this.position);
-        if ((charCode >= 65 && charCode <= 90) || // A-Z
-            (charCode >= 97 && charCode <= 122) || // a-z
-            (charCode >= 48 && charCode <= 57) || // 0-9
-            charCode === 95) { // _
-          this.advance();
-        } else {
-          break;
-        }
-      }
+      // Consume head and the rest of identifier
+      this.advance();
+      this.scanIdentifierBody();
       
       const value = this.input.substring(start, this.position);
       return this.createToken(TokenType.ENVIRONMENT_VARIABLE, value, start, this.position, startLine, startColumn);
     }
   }
   
-  private readString(quote: string): Token {
+  private readString(quote: "'" | '"'): Token {
     const start = this.position;
     const startLine = this.line;
     const startColumn = this.column;
-    
-    this.advance(); // Skip opening quote
-    
-    while (this.position < this.input.length) {
-      const char = this.current();
-      
-      if (char === quote) {
-        this.advance(); // Skip closing quote
-        const value = this.input.substring(start, this.position);
-        return this.createToken(TokenType.STRING, value, start, this.position, startLine, startColumn);
-      }
-      
-      if (char === '\\') {
-        this.advance(); // Skip escape character
-        if (this.position >= this.input.length) {
-          throw this.error('Unterminated string');
-        }
-        // Skip the escaped character
-        this.advance();
-      } else {
-        this.advance();
-      }
-    }
-    
-    throw this.error('Unterminated string');
+    this.scanQuoted(quote, 'Unterminated string');
+    const value = this.input.substring(start, this.position);
+    return this.createToken(TokenType.STRING, value, start, this.position, startLine, startColumn);
   }
   
   private readNumber(): Token {
@@ -545,6 +383,31 @@ export class Lexer {
       }
     }
     
+    // Look ahead for a single-quoted unit to form a Quantity literal
+    // Pattern: <number>[ \t]?'<unit>' (no newline between number and unit)
+    const i = this.position;
+    let j = i;
+    // allow spaces/tabs only; do not cross lines
+    while (j < this.input.length) {
+      const ch = this.input[j]!;
+      if (ch === ' ' || ch === '\t') {
+        j++;
+        continue;
+      }
+      break;
+    }
+
+    if (j < this.input.length && this.input[j] === "'") {
+      // We will consume optional spaces/tabs and the quoted unit, then emit QUANTITY token
+      while (this.position < j) {
+        this.advance();
+      }
+      // Current is the opening quote; scan quoted unit including escapes
+      this.scanQuoted("'", 'Unterminated string');
+      const quantityValue = this.input.substring(start, this.position);
+      return this.createToken(TokenType.QUANTITY, quantityValue, start, this.position, startLine, startColumn);
+    }
+
     const value = this.input.substring(start, this.position);
     return this.createToken(TokenType.NUMBER, value, start, this.position, startLine, startColumn);
   }
@@ -574,6 +437,8 @@ export class Lexer {
       this.advance();
     }
     
+    let hasTime = false;
+    
     // Optional month, day, time parts
     if (this.current() === '-') {
       this.advance();
@@ -597,17 +462,31 @@ export class Lexer {
       }
     }
     
-    // Optional time part
+    // Check for time part or T suffix
     if (this.current() === 'T') {
+      hasTime = true;
       this.advance();
-      this.readTimeFormat();
+      // Check if there's actual time content after T
+      if (this.isDigit(this.current())) {
+        this.readTimeFormat();
+      }
+      // else: it's just a T suffix like @2020T or @2020-01T
     }
     
-    // Optional timezone
-    this.readTimezone();
+    // Optional timezone (only if we have time component with actual time values)
+    if (hasTime && this.position > 0 && this.input[this.position - 1] !== 'T') {
+      // Only read timezone if there's actual time content after T
+      this.readTimezone();
+    }
     
     const value = this.input.substring(start, this.position);
-    return this.createToken(TokenType.DATETIME, value, start, this.position, startLine, startColumn);
+    
+    // Determine token type based on content
+    // DateTime: has 'T' anywhere
+    // Date: no 'T' at all
+    const tokenType = hasTime ? TokenType.DATETIME : TokenType.DATE;
+    
+    return this.createToken(tokenType, value, start, this.position, startLine, startColumn);
   }
   
   private readTime(start: number, startLine: number, startColumn: number): Token {
@@ -648,11 +527,11 @@ export class Lexer {
         }
         
         // Optional milliseconds
-        if (this.current() === '.') {
+        // Only consume the period if it's followed by a digit
+        // This prevents consuming periods that are method calls like .toDate()
+        if (this.current() === '.' && this.position + 1 < this.input.length && 
+            this.isDigit(this.input[this.position + 1]!)) {
           this.advance();
-          if (!this.isDigit(this.current())) {
-            throw this.error('Invalid time format');
-          }
           while (this.isDigit(this.current())) {
             this.advance();
           }
@@ -686,6 +565,82 @@ export class Lexer {
       }
     }
   }
+
+  // Operator utilities
+  private static readonly OPERATORS: readonly string[] = [
+    '!=', '!~', '<=', '>=',
+    '+', '-', '*', '/', '<', '>', '=', '~', '|', '&',
+  ];
+
+  private isOperatorStarter(ch: string): boolean {
+    switch (ch) {
+      case '+':
+      case '-':
+      case '*':
+      case '/':
+      case '<':
+      case '>':
+      case '=':
+      case '!':
+      case '~':
+      case '|':
+      case '&':
+        return true;
+      default:
+        return false;
+    }
+  }
+
+  private readOperator(start: number, startLine: number, startColumn: number): Token {
+    // Try longest-first
+    for (const op of Lexer.OPERATORS) {
+      const end = start + op.length;
+      if (this.input.startsWith(op, start)) {
+        // Advance to end and emit
+        while (this.position < end) {
+          this.advance();
+        }
+        return this.createToken(TokenType.OPERATOR, op, start, this.position, startLine, startColumn);
+      }
+    }
+    // Special-case: lone '!'
+    if (this.input[start] === '!') {
+      // consume '!' to keep position consistent with old behavior before throwing?
+      // Old code threw without consuming, but tests only assert message content.
+      // Preserve position by not advancing.
+      throw this.error(`Unexpected character '!' at position ${start}`);
+    }
+    // Fallback: treat as unexpected
+    const ch = this.input[start] ?? '';
+    throw this.error(`Unexpected character '${ch}' at position ${start}`);
+  }
+
+  // Generic quoted scanner used by strings, delimited identifiers, env vars, and quantity units
+  private scanQuoted(quote: "'" | '"' | '`', unterminatedMessage: string): void {
+    if (this.current() !== quote) {
+      throw this.error('Internal error: scanQuoted called at non-quote');
+    }
+    // Skip opening quote
+    this.advance();
+    while (this.position < this.input.length) {
+      const ch = this.current();
+      if (ch === quote) {
+        this.advance(); // Skip closing quote
+        return;
+      }
+      if (ch === '\\') {
+        this.advance();
+        if (this.position >= this.input.length) {
+          throw this.error(unterminatedMessage);
+        }
+        // Skip escaped character
+        this.advance();
+        continue;
+      }
+      this.advance();
+    }
+    throw this.error(unterminatedMessage);
+  }
   
   private skipWhitespace(): void {
     while (this.position < this.input.length) {
@@ -733,8 +688,6 @@ export class Lexer {
         if (char === '\n') {
           this.line++;
           this.column = 1;
-          this.lspLine++;
-          this.lspCharacter = 0;
         } else if (char === '\r') {
           // Handle \r\n as single line ending
           if (this.position + 1 < this.input.length && this.input[this.position + 1] === '\n') {
@@ -743,12 +696,9 @@ export class Lexer {
             // Standalone \r
             this.line++;
             this.column = 1;
-            this.lspLine++;
-            this.lspCharacter = 0;
           }
         } else {
           this.column++;
-          this.lspCharacter++;
         }
       }
       this.position++;
@@ -773,6 +723,28 @@ export class Lexer {
     if (!char) return false;
     return char === ' ' || char === '\t' || char === '\n' || char === '\r';
   }
+
+  // Identifier helpers (ASCII-based)
+  private isIdentifierHead(code: number): boolean {
+    return (code >= 65 && code <= 90) || // A-Z
+           (code >= 97 && code <= 122) || // a-z
+           code === 95; // _
+  }
+
+  private isIdentifierPart(code: number): boolean {
+    return this.isIdentifierHead(code) || (code >= 48 && code <= 57); // 0-9
+  }
+
+  private scanIdentifierBody(): void {
+    while (this.position < this.input.length) {
+      const code = this.input.charCodeAt(this.position);
+      if (this.isIdentifierPart(code)) {
+        this.advance();
+      } else {
+        break;
+      }
+    }
+  }
   
   /**
    * Convert absolute offset to LSP Position
@@ -833,6 +805,12 @@ export class Lexer {
   }
   
   private error(message: string): Error {
+    if (this.options.trackPosition) {
+      const pos = this.offsetToPosition(this.position);
+      const line = pos.line + 1; // present as 1-based
+      const col = pos.character + 1; // present as 1-based
+      return new Error(`Lexer error at ${line}:${col}: ${message}`);
+    }
     return new Error(`Lexer error: ${message}`);
   }
   
@@ -878,4 +856,4 @@ export function isOperatorValue(token: Token, value: string): boolean {
 // Helper to check if a token is an environment variable
 export function isEnvironmentVariable(token: Token): boolean {
   return token.type === TokenType.ENVIRONMENT_VARIABLE;
-}
+}