@higher.archi/boe 1.0.0

package/src/core/parser/tokenizer/index.ts

@@ -0,0 +1,85 @@
+import type { OperatorTable, Token } from '../types';
+import { createScanner, currentChar, isAtEnd } from './scanner';
+import {
+  readIdentifier,
+  readNumber,
+  readPropertyRef,
+  readString,
+  readSymbol,
+  skipWhitespace
+} from './readers';
+import { TokenizeError } from '../errors';
+
+/**
+ * Main tokenization function.
+ *
+ * Converts an input string into a stream of tokens that the parser
+ * can process. Uses modular readers that each handle a specific
+ * token type.
+ *
+ * Reader Priority (Order Matters!):
+ * --------------------------------
+ * 1. Whitespace - skip it (doesn't produce tokens)
+ * 2. Property ref - starts with $
+ * 3. String - starts with ' or "
+ * 4. Number - starts with digit, or - followed by digit (in right context)
+ * 5. Symbol - operators like >, >=, (, )
+ * 6. Identifier - words like 'and', 'contains', 'daysAgo'
+ *
+ * Why this order?
+ * - Property refs start with $ which is unique
+ * - Strings start with quotes which is unique
+ * - Numbers are checked before symbols so -5 is a number, not minus then 5
+ * - Symbols before identifiers so >= is one token, not > followed by =
+ * - Identifiers last as the catch-all for words
+ *
+ * @param input - The expression string to tokenize
+ * @param operatorTable - Map of operator aliases to their info
+ * @returns Array of tokens ending with EOF
+ */
+export function tokenize(input: string, operatorTable: OperatorTable): Token[] {
+  const tokens: Token[] = [];
+  let state = createScanner(input);
+
+  while (!isAtEnd(state)) {
+    // Skip any whitespace
+    state = skipWhitespace(state);
+    if (isAtEnd(state)) break;
+
+    // Get the last token (for number reader context)
+    const lastToken = tokens.length > 0 ? tokens[tokens.length - 1] : undefined;
+
+    // Try each reader in priority order
+    const result =
+      readPropertyRef(state) ||
+      readString(state) ||
+      readNumber(state, lastToken) ||
+      readSymbol(state, operatorTable) ||
+      readIdentifier(state, operatorTable);
+
+    if (!result) {
+      // No reader matched - unknown character
+      throw new TokenizeError(
+        `Unexpected character: '${currentChar(state)}'`,
+        state.position,
+        input
+      );
+    }
+
+    tokens.push(result.token);
+    state = result.newState;
+  }
+
+  // Add EOF marker
+  tokens.push({
+    type: 'EOF',
+    value: '',
+    position: state.position
+  });
+
+  return tokens;
+}
+
+// Re-export types for convenience
+export type { ScannerState, ReaderResult } from './types';
+export { createScanner } from './scanner';

package/src/core/parser/tokenizer/readers/identifier.reader.ts

@@ -0,0 +1,118 @@
+import type { OperatorTable, TokenType } from '../../types';
+import type { ReaderResult, ScannerState } from '../types';
+import { advance, isAtEnd, isIdentifierPart, isIdentifierStart, peek } from '../scanner';
+import { skipWhitespace } from './whitespace.reader';
+
+/**
+ * Reserved words that map to literal values.
+ * These are always interpreted as their literal meaning,
+ * never as operators or functions.
+ */
+const LITERALS: Record<string, TokenType> = {
+  true: 'BOOLEAN',
+  false: 'BOOLEAN',
+  null: 'NULL'
+};
+
+/**
+ * Reads word-based tokens: operators, functions, booleans, null.
+ *
+ * Classification logic determines what kind of token a word is:
+ *   1. 'true'/'false' → BOOLEAN literal
+ *   2. 'null' → NULL literal
+ *   3. Known operator word (and, or, contains, gt, etc.) → OPERATOR
+ *   4. Any other word followed by '(' → FUNCTION
+ *   5. Otherwise → treat as FUNCTION (parser will validate)
+ *
+ * Examples:
+ *   and        → OPERATOR('and', canonical: 'and')
+ *   contains   → OPERATOR('contains', canonical: 'contains')
+ *   gt         → OPERATOR('gt', canonical: 'greaterThan')
+ *   true       → BOOLEAN('true')
+ *   false      → BOOLEAN('false')
+ *   null       → NULL('null')
+ *   daysAgo(   → FUNCTION('daysAgo')
+ *   now(       → FUNCTION('now')
+ *
+ * Note on Unknown Words:
+ * ----------------------
+ * Words that aren't recognized operators might be function names.
+ * We check if the word is followed by '(' to classify it as FUNCTION.
+ * This allows custom functions to be used without modification.
+ */
+export function readIdentifier(
+  state: ScannerState,
+  operatorTable: OperatorTable
+): ReaderResult | null {
+  // Must start with a letter or underscore
+  if (!isIdentifierStart(peek(state))) {
+    return null;
+  }
+
+  const startPosition = state.position;
+
+  // Read the full identifier
+  let value = '';
+  while (!isAtEnd(state) && isIdentifierPart(peek(state))) {
+    value += peek(state);
+    state = advance(state);
+  }
+
+  // Check if it's a literal (true, false, null)
+  const literalType = LITERALS[value];
+  if (literalType) {
+    return {
+      token: {
+        type: literalType,
+        value,
+        position: startPosition
+      },
+      newState: state
+    };
+  }
+
+  // Check if followed by '(' - this makes it a function call
+  // This check must come BEFORE operator check so that gt($a, 5)
+  // treats 'gt' as a function, not an operator
+  const stateAfterWhitespace = skipWhitespace(state);
+  if (peek(stateAfterWhitespace) === '(') {
+    // It's a function call - use canonical name from operator table if available
+    const operatorInfo = operatorTable.get(value);
+    return {
+      token: {
+        type: 'FUNCTION',
+        value,
+        canonical: operatorInfo?.canonical,
+        position: startPosition
+      },
+      newState: state // Don't consume whitespace/paren - let tokenizer handle
+    };
+  }
+
+  // Check if it's a known operator (in infix position)
+  const operatorInfo = operatorTable.get(value);
+  if (operatorInfo) {
+    return {
+      token: {
+        type: 'OPERATOR',
+        value,
+        canonical: operatorInfo.canonical,
+        position: startPosition
+      },
+      newState: state
+    };
+  }
+
+  // Unknown identifier that's not followed by ( and not a known operator
+  // Treat it as a function anyway - parser will provide better error
+  // if used incorrectly. This allows forward-compatibility with new
+  // operators/functions without updating the parser.
+  return {
+    token: {
+      type: 'FUNCTION',
+      value,
+      position: startPosition
+    },
+    newState: state
+  };
+}

package/src/core/parser/tokenizer/readers/index.ts

@@ -0,0 +1,20 @@
+/**
+ * Token Readers
+ *
+ * Each reader is responsible for recognizing and consuming a specific
+ * type of token from the input stream. They are pure functions that:
+ *
+ *   1. Take a scanner state (and optionally context like operator table)
+ *   2. Return null if they don't match at the current position
+ *   3. Return { token, newState } if they do match
+ *
+ * The main tokenizer tries readers in a specific order until one matches.
+ * This module exports all readers for use by the tokenizer.
+ */
+
+export { skipWhitespace } from './whitespace.reader';
+export { readPropertyRef } from './property-ref.reader';
+export { readString } from './string.reader';
+export { readNumber } from './number.reader';
+export { readSymbol } from './symbol.reader';
+export { readIdentifier } from './identifier.reader';

package/src/core/parser/tokenizer/readers/number.reader.ts

@@ -0,0 +1,132 @@
+import type { Token } from '../../types';
+import type { ReaderResult, ScannerState } from '../types';
+import { advance, isAtEnd, isDigit, peek, peekAhead } from '../scanner';
+
+/**
+ * Reads numeric literals (integers, decimals, and percentages).
+ *
+ * Supports:
+ *   - Integers: 42, 0, 123
+ *   - Decimals: 3.14, 0.5, 123.456
+ *   - Negative numbers: -5, -3.14 (when context appropriate)
+ *   - Positive numbers: +5, +3.14 (when context appropriate)
+ *   - Percentages: 50%, -50%, +300%, 0.5%
+ *
+ * Examples:
+ *   42      → NUMBER(42)
+ *   3.14    → NUMBER(3.14)
+ *   -5      → NUMBER(-5)  // only when used as value, not subtraction
+ *   +5      → NUMBER(+5)  // only when used as value, not addition
+ *   0.5     → NUMBER(0.5)
+ *   50%     → NUMBER(50%) // percentage literal
+ *   -50%    → NUMBER(-50%)
+ *   +300%   → NUMBER(+300%)
+ *
+ * Note on Sign Handling:
+ * ----------------------
+ * The +/- signs are ambiguous: "$a - 5" is subtraction, but
+ * "gt(-5, $x)" should treat -5 as a negative number literal.
+ *
+ * We handle signs in the number reader when:
+ *   - At the start of input
+ *   - After an opening paren or comma
+ *   - After an operator
+ *
+ * Note on Percentages:
+ * --------------------
+ * When `%` immediately follows digits (no space), it's a percentage literal.
+ * With spaces or operands, `%` is the modulo operator:
+ *   - 50%     → percentage (0.5)
+ *   - 50 % 2  → modulo operation
+ *
+ * The tokenizer provides context about what came before.
+ */
+export function readNumber(
+  state: ScannerState,
+  lastToken?: Token
+): ReaderResult | null {
+  const char = peek(state);
+  const startPosition = state.position;
+
+  // Check if this is a potential number
+  let sign = '';
+
+  if (char === '-' || char === '+') {
+    // Sign character - could be sign or arithmetic operator
+    // It's a sign if:
+    //   - At start of input (no previous token)
+    //   - After ( or ,
+    //   - After an operator
+    const isSignContext =
+      !lastToken ||
+      lastToken.type === 'LPAREN' ||
+      lastToken.type === 'COMMA' ||
+      lastToken.type === 'OPERATOR';
+
+    if (!isSignContext) {
+      // This should be treated as arithmetic operator
+      return null;
+    }
+
+    // Check if followed by a digit
+    const nextChar = peekAhead(state, 1);
+    if (!isDigit(nextChar)) {
+      // Not a number (might be unary operator)
+      return null;
+    }
+
+    sign = char;
+    state = advance(state); // consume the sign
+  } else if (!isDigit(char)) {
+    // Doesn't start with digit or sign
+    return null;
+  }
+
+  // Read the integer part
+  let value = sign;
+  value += readDigits(state).value;
+  state = readDigits(state).newState;
+
+  // Check for decimal point
+  if (peek(state) === '.' && isDigit(peekAhead(state, 1))) {
+    value += '.';
+    state = advance(state); // consume the dot
+
+    // Read fractional part
+    const fractional = readDigits(state);
+    value += fractional.value;
+    state = fractional.newState;
+  }
+
+  // Check for percentage suffix (greedy: 50% is percentage, not modulo)
+  if (peek(state) === '%') {
+    value += '%';
+    state = advance(state); // consume the percent sign
+  }
+
+  return {
+    token: {
+      type: 'NUMBER',
+      value,
+      position: startPosition
+    },
+    newState: state
+  };
+}
+
+/**
+ * Reads a sequence of digit characters.
+ * Returns the digits as a string and the updated state.
+ */
+function readDigits(
+  state: ScannerState
+): { value: string; newState: ScannerState } {
+  let value = '';
+
+  while (!isAtEnd(state) && isDigit(peek(state))) {
+    value += peek(state);
+    state = advance(state);
+  }
+
+  return { value, newState: state };
+}

package/src/core/parser/tokenizer/readers/property-ref.reader.ts

@@ -0,0 +1,130 @@
+import type { ReaderResult, ScannerState } from '../types';
+import { advance, isAtEnd, isIdentifierPart, isIdentifierStart, peek } from '../scanner';
+
+/**
+ * Reads property references that start with $.
+ *
+ * Grammar: '$' identifier ('.' identifier)*
+ *
+ * Property references are used to access values from the fact context.
+ * They support dot notation for nested property access.
+ *
+ * Examples:
+ *   $age            → PROPERTY_REF('$age')
+ *   $user.name      → PROPERTY_REF('$user.name')
+ *   $user.address.city → PROPERTY_REF('$user.address.city')
+ *   $item.0         → PROPERTY_REF('$item.0')  // array index
+ *
+ * Note: The $ prefix distinguishes property refs from other identifiers.
+ * Without it, "age" would be ambiguous (operator? function? value?).
+ */
+export function readPropertyRef(state: ScannerState): ReaderResult | null {
+  // Must start with $
+  if (peek(state) !== '$') {
+    return null;
+  }
+
+  const startPosition = state.position;
+  let value = '$';
+  state = advance(state); // consume the $
+
+  // Read the first identifier segment
+  // Note: After $, we need at least one valid identifier character
+  if (isAtEnd(state) || !isValidPropertyStart(peek(state))) {
+    // Just a lone $ - this will be caught as an error by the tokenizer
+    return null;
+  }
+
+  // Read first segment (the main property name)
+  const firstSegment = readIdentifierSegment(state);
+  value += firstSegment.value;
+  state = firstSegment.newState;
+
+  // Read any additional dot-separated segments
+  // These can be property names OR numeric indices (for arrays)
+  while (!isAtEnd(state) && peek(state) === '.') {
+    value += '.';
+    state = advance(state); // consume the dot
+
+    // After a dot, we need another segment
+    if (isAtEnd(state)) {
+      // Trailing dot - include it and let parser handle the error
+      break;
+    }
+
+    const segment = readPropertySegment(state);
+    if (segment.value.length === 0) {
+      // Empty segment after dot - let parser handle
+      break;
+    }
+
+    value += segment.value;
+    state = segment.newState;
+  }
+
+  return {
+    token: {
+      type: 'PROPERTY_REF',
+      value,
+      position: startPosition
+    },
+    newState: state
+  };
+}
+
+/**
+ * Checks if a character can start a property name after $.
+ * Property names start with letter or underscore (not digit).
+ */
+function isValidPropertyStart(char: string): boolean {
+  return isIdentifierStart(char);
+}
+
+/**
+ * Reads a single identifier segment (letters, digits, underscores).
+ */
+function readIdentifierSegment(
+  state: ScannerState
+): { value: string; newState: ScannerState } {
+  let value = '';
+
+  while (!isAtEnd(state) && isIdentifierPart(peek(state))) {
+    value += peek(state);
+    state = advance(state);
+  }
+
+  return { value, newState: state };
+}
+
+/**
+ * Reads a property segment after a dot.
+ * Can be an identifier OR a numeric index.
+ *
+ * Examples:
+ *   .name  → 'name' (property)
+ *   .0     → '0' (array index)
+ *   .items → 'items' (property)
+ */
+function readPropertySegment(
+  state: ScannerState
+): { value: string; newState: ScannerState } {
+  let value = '';
+  const char = peek(state);
+
+  // Check if it's a digit (array index) or identifier start
+  if (char >= '0' && char <= '9') {
+    // Read numeric index
+    while (!isAtEnd(state) && peek(state) >= '0' && peek(state) <= '9') {
+      value += peek(state);
+      state = advance(state);
+    }
+  } else if (isIdentifierStart(char)) {
+    // Read identifier
+    while (!isAtEnd(state) && isIdentifierPart(peek(state))) {
+      value += peek(state);
+      state = advance(state);
+    }
+  }
+
+  return { value, newState: state };
+}

package/src/core/parser/tokenizer/readers/string.reader.ts

@@ -0,0 +1,110 @@
+import type { ReaderResult, ScannerState } from '../types';
+import { advance, isAtEnd, peek } from '../scanner';
+import { TokenizeError } from '../../errors';
+
+/**
+ * Reads string literals with single or double quotes.
+ *
+ * Supports common escape sequences:
+ *   \'  - single quote
+ *   \"  - double quote
+ *   \\  - backslash
+ *   \n  - newline
+ *   \t  - tab
+ *   \r  - carriage return
+ *
+ * Examples:
+ *   'hello'     → STRING('hello')
+ *   "world"     → STRING('world')
+ *   'it\'s'     → STRING("it's")
+ *   "say \"hi\"" → STRING('say "hi"')
+ *
+ * The returned value is the unescaped string content (without quotes).
+ */
+export function readString(state: ScannerState): ReaderResult | null {
+  const quote = peek(state);
+
+  // Must start with a quote character
+  if (quote !== '"' && quote !== "'") {
+    return null;
+  }
+
+  const startPosition = state.position;
+  state = advance(state); // consume opening quote
+
+  let value = '';
+
+  while (!isAtEnd(state)) {
+    const char = peek(state);
+
+    // Found the closing quote - we're done
+    if (char === quote) {
+      state = advance(state); // consume closing quote
+      return {
+        token: {
+          type: 'STRING',
+          value,
+          position: startPosition
+        },
+        newState: state
+      };
+    }
+
+    // Handle escape sequences
+    if (char === '\\') {
+      state = advance(state); // consume backslash
+
+      if (isAtEnd(state)) {
+        throw new TokenizeError(
+          'Unterminated string: unexpected end after escape character',
+          state.position,
+          state.input
+        );
+      }
+
+      const escaped = peek(state);
+      value += resolveEscape(escaped);
+      state = advance(state);
+      continue;
+    }
+
+    // Regular character - add to value
+    value += char;
+    state = advance(state);
+  }
+
+  // Reached end of input without closing quote
+  throw new TokenizeError(
+    `Unterminated string literal starting with ${quote}`,
+    startPosition,
+    state.input
+  );
+}
+
+/**
+ * Resolves an escape sequence to its actual character.
+ *
+ * For unknown escape sequences, we return the character as-is
+ * (with the backslash consumed). This is lenient behavior that
+ * allows things like \x to just be 'x'.
+ */
+function resolveEscape(char: string): string {
+  switch (char) {
+    case "'":
+      return "'";
+    case '"':
+      return '"';
+    case '\\':
+      return '\\';
+    case 'n':
+      return '\n';
+    case 't':
+      return '\t';
+    case 'r':
+      return '\r';
+    default:
+      // Unknown escape - be lenient and return the character
+      // This allows '\x' to become 'x' rather than throwing an error
+      return char;
+  }
+}