@tradik/xslt-processor 1.0.0

package/src/xpath/tokenizer.js

@@ -0,0 +1,383 @@
+/**
+ * XPath 1.0 Tokenizer
+ * Based on W3C XPath 1.0 Specification: http://www.w3.org/TR/1999/REC-xpath-19991116
+ *
+ * Converts XPath expression strings into token streams for parsing.
+ */
+
+"use strict";
+
+export const TokenType = {
+  // Literals
+  NUMBER: "NUMBER",
+  LITERAL: "LITERAL",
+
+  // Operators
+  SLASH: "SLASH",
+  DOUBLE_SLASH: "DOUBLE_SLASH",
+  PIPE: "PIPE",
+  PLUS: "PLUS",
+  MINUS: "MINUS",
+  STAR: "STAR",
+  DIV: "DIV",
+  MOD: "MOD",
+  EQUALS: "EQUALS",
+  NOT_EQUALS: "NOT_EQUALS",
+  LT: "LT",
+  LTE: "LTE",
+  GT: "GT",
+  GTE: "GTE",
+  AND: "AND",
+  OR: "OR",
+
+  // Brackets
+  LPAREN: "LPAREN",
+  RPAREN: "RPAREN",
+  LBRACKET: "LBRACKET",
+  RBRACKET: "RBRACKET",
+
+  // Axes
+  AXIS: "AXIS",
+  AT: "AT",
+  DOT: "DOT",
+  DOUBLE_DOT: "DOUBLE_DOT",
+  DOUBLE_COLON: "DOUBLE_COLON",
+
+  // Names
+  NAME: "NAME",
+  NCNAME: "NCNAME",
+  PREFIX: "PREFIX",
+  FUNCTION: "FUNCTION",
+  NODE_TYPE: "NODE_TYPE",
+
+  // Other
+  COMMA: "COMMA",
+  DOLLAR: "DOLLAR",
+  COLON: "COLON",
+  EOF: "EOF",
+};
+
+const AXIS_NAMES = new Set([
+  "ancestor",
+  "ancestor-or-self",
+  "attribute",
+  "child",
+  "descendant",
+  "descendant-or-self",
+  "following",
+  "following-sibling",
+  "namespace",
+  "parent",
+  "preceding",
+  "preceding-sibling",
+  "self",
+]);
+
+const NODE_TYPES = new Set([
+  "comment",
+  "text",
+  "processing-instruction",
+  "node",
+]);
+
+const OPERATORS = new Set(["and", "or", "mod", "div"]);
+
+export class Token {
+  constructor(type, value, position) {
+    this.type = type;
+    this.value = value;
+    this.position = position;
+  }
+
+  toString() {
+    return `Token(${this.type}, ${JSON.stringify(this.value)}, pos=${this.position})`;
+  }
+}
+
+// Tokens that indicate the NEXT token should NOT be treated as an OperatorName
+// Per XPath 1.0 spec: "If there is a preceding token and the preceding token is not one of
+// @, ::, (, [, , or an Operator, then... an NCName must be recognized as an OperatorName."
+// So after these tokens, OperatorNames (div, mod, and, or) should be treated as regular names.
+const OPERATOR_CONTEXT_BLOCKERS = new Set([
+  TokenType.AT,
+  TokenType.DOUBLE_COLON,
+  TokenType.LPAREN,
+  TokenType.LBRACKET,
+  TokenType.COMMA,
+  // Operators - after an operator, next name token should be a name, not an operator
+  TokenType.SLASH,
+  TokenType.DOUBLE_SLASH,
+  TokenType.PIPE,
+  TokenType.PLUS,
+  TokenType.MINUS,
+  TokenType.STAR,
+  TokenType.DIV,
+  TokenType.MOD,
+  TokenType.EQUALS,
+  TokenType.NOT_EQUALS,
+  TokenType.LT,
+  TokenType.LTE,
+  TokenType.GT,
+  TokenType.GTE,
+  TokenType.AND,
+  TokenType.OR,
+]);
+
+export class XPathTokenizer {
+  constructor(expression) {
+    this.expression = expression;
+    this.position = 0;
+    this.tokens = [];
+  }
+
+  /**
+   * Get the last token that was added (if any)
+   */
+  getLastToken() {
+    return this.tokens.length > 0 ? this.tokens[this.tokens.length - 1] : null;
+  }
+
+  /**
+   * Check if the current context allows operator names (div, mod, and, or)
+   * Per XPath 1.0 spec disambiguation rules.
+   */
+  isOperatorContext() {
+    const lastToken = this.getLastToken();
+    // No preceding token means we're at the start - not an operator context
+    if (!lastToken) return false;
+    // If preceding token is a "blocker", it's not an operator context
+    if (OPERATOR_CONTEXT_BLOCKERS.has(lastToken.type)) return false;
+    // Otherwise, it IS an operator context (after names, numbers, closing brackets, etc.)
+    return true;
+  }
+
+  tokenize() {
+    this.tokens = [];
+    this.position = 0;
+
+    while (this.position < this.expression.length) {
+      this.skipWhitespace();
+      if (this.position >= this.expression.length) break;
+
+      const token = this.nextToken();
+      if (token) {
+        this.tokens.push(token);
+      }
+    }
+
+    this.tokens.push(new Token(TokenType.EOF, null, this.position));
+    return this.tokens;
+  }
+
+  skipWhitespace() {
+    while (
+      this.position < this.expression.length &&
+      /\s/.test(this.expression[this.position])
+    ) {
+      this.position++;
+    }
+  }
+
+  peek(offset = 0) {
+    return this.expression[this.position + offset];
+  }
+
+  consume() {
+    return this.expression[this.position++];
+  }
+
+  nextToken() {
+    const startPos = this.position;
+    const char = this.peek();
+
+    // String literals
+    if (char === '"' || char === "'") {
+      return this.readStringLiteral();
+    }
+
+    // Numbers
+    if (/[0-9]/.test(char) || (char === "." && /[0-9]/.test(this.peek(1)))) {
+      return this.readNumber();
+    }
+
+    // Multi-character operators
+    if (char === "/" && this.peek(1) === "/") {
+      this.position += 2;
+      return new Token(TokenType.DOUBLE_SLASH, "//", startPos);
+    }
+
+    if (char === "." && this.peek(1) === ".") {
+      this.position += 2;
+      return new Token(TokenType.DOUBLE_DOT, "..", startPos);
+    }
+
+    if (char === ":" && this.peek(1) === ":") {
+      this.position += 2;
+      return new Token(TokenType.DOUBLE_COLON, "::", startPos);
+    }
+
+    if (char === "!" && this.peek(1) === "=") {
+      this.position += 2;
+      return new Token(TokenType.NOT_EQUALS, "!=", startPos);
+    }
+
+    if (char === "<" && this.peek(1) === "=") {
+      this.position += 2;
+      return new Token(TokenType.LTE, "<=", startPos);
+    }
+
+    if (char === ">" && this.peek(1) === "=") {
+      this.position += 2;
+      return new Token(TokenType.GTE, ">=", startPos);
+    }
+
+    // Single-character tokens
+    const singleCharTokens = {
+      "/": TokenType.SLASH,
+      "|": TokenType.PIPE,
+      "+": TokenType.PLUS,
+      "-": TokenType.MINUS,
+      "*": TokenType.STAR,
+      "=": TokenType.EQUALS,
+      "<": TokenType.LT,
+      ">": TokenType.GT,
+      "(": TokenType.LPAREN,
+      ")": TokenType.RPAREN,
+      "[": TokenType.LBRACKET,
+      "]": TokenType.RBRACKET,
+      "@": TokenType.AT,
+      ".": TokenType.DOT,
+      ",": TokenType.COMMA,
+      $: TokenType.DOLLAR,
+      ":": TokenType.COLON,
+    };
+
+    if (singleCharTokens[char]) {
+      this.position++;
+      return new Token(singleCharTokens[char], char, startPos);
+    }
+
+    // Names and keywords
+    if (this.isNameStartChar(char)) {
+      return this.readName();
+    }
+
+    throw new Error(
+      `Unexpected character '${char}' at position ${this.position} in expression: ${this.expression}`,
+    );
+  }
+
+  readStringLiteral() {
+    const startPos = this.position;
+    const quote = this.consume();
+    let value = "";
+
+    while (this.position < this.expression.length && this.peek() !== quote) {
+      value += this.consume();
+    }
+
+    if (this.position >= this.expression.length) {
+      throw new Error(`Unterminated string literal at position ${startPos}`);
+    }
+
+    this.consume(); // closing quote
+    return new Token(TokenType.LITERAL, value, startPos);
+  }
+
+  readNumber() {
+    const startPos = this.position;
+    let value = "";
+
+    // Integer part
+    while (
+      this.position < this.expression.length &&
+      /[0-9]/.test(this.peek())
+    ) {
+      value += this.consume();
+    }
+
+    // Decimal part (handles both 1.5 and .5 style numbers)
+    // For .5 style: entry condition ensures digit follows, so this branch handles it
+    if (this.peek() === "." && /[0-9]/.test(this.peek(1))) {
+      value += this.consume(); // .
+      while (
+        this.position < this.expression.length &&
+        /[0-9]/.test(this.peek())
+      ) {
+        value += this.consume();
+      }
+    }
+
+    return new Token(TokenType.NUMBER, parseFloat(value), startPos);
+  }
+
+  readName() {
+    const startPos = this.position;
+    let value = "";
+
+    while (
+      this.position < this.expression.length &&
+      this.isNameChar(this.peek())
+    ) {
+      value += this.consume();
+    }
+
+    // Check for axis name followed by ::
+    this.skipWhitespace();
+    if (AXIS_NAMES.has(value) && this.peek() === ":" && this.peek(1) === ":") {
+      return new Token(TokenType.AXIS, value, startPos);
+    }
+
+    // Check for function call (followed by '(')
+    const savedPos = this.position;
+    this.skipWhitespace();
+    if (this.peek() === "(") {
+      // Check if it's a node type test
+      if (NODE_TYPES.has(value)) {
+        return new Token(TokenType.NODE_TYPE, value, startPos);
+      }
+      return new Token(TokenType.FUNCTION, value, startPos);
+    }
+    this.position = savedPos;
+
+    // Check for operators - only in operator context per XPath 1.0 disambiguation rules
+    if (OPERATORS.has(value) && this.isOperatorContext()) {
+      const opTokens = {
+        and: TokenType.AND,
+        or: TokenType.OR,
+        mod: TokenType.MOD,
+        div: TokenType.DIV,
+      };
+      return new Token(opTokens[value], value, startPos);
+    }
+
+    return new Token(TokenType.NAME, value, startPos);
+  }
+
+  isNameStartChar(char) {
+    if (!char) return false;
+    const code = char.charCodeAt(0);
+    return (
+      char === "_" ||
+      (code >= 65 && code <= 90) || // A-Z
+      (code >= 97 && code <= 122) || // a-z
+      code >= 0xc0 // Unicode letters
+    );
+  }
+
+  isNameChar(char) {
+    if (!char) return false;
+    const code = char.charCodeAt(0);
+    return (
+      this.isNameStartChar(char) ||
+      char === "-" ||
+      char === "." ||
+      (code >= 48 && code <= 57) // 0-9
+    );
+  }
+}
+
+export function tokenize(expression) {
+  const tokenizer = new XPathTokenizer(expression);
+  return tokenizer.tokenize();
+}

package/src/xpath/tokenizer.test.js

@@ -0,0 +1,224 @@
+/**
+ * XPath Tokenizer Tests
+ */
+
+import { describe, it } from "node:test";
+import assert from "node:assert";
+import { tokenize, TokenType } from "./tokenizer.js";
+
+describe("XPath Tokenizer", () => {
+  describe("Simple paths", () => {
+    it("should tokenize simple element name", () => {
+      const tokens = tokenize("foo");
+      assert.strictEqual(tokens.length, 2);
+      assert.strictEqual(tokens[0].type, TokenType.NAME);
+      assert.strictEqual(tokens[0].value, "foo");
+      assert.strictEqual(tokens[1].type, TokenType.EOF);
+    });
+
+    it("should tokenize root path", () => {
+      const tokens = tokenize("/");
+      assert.strictEqual(tokens[0].type, TokenType.SLASH);
+    });
+
+    it("should tokenize absolute path", () => {
+      const tokens = tokenize("/foo/bar");
+      assert.strictEqual(tokens[0].type, TokenType.SLASH);
+      assert.strictEqual(tokens[1].type, TokenType.NAME);
+      assert.strictEqual(tokens[1].value, "foo");
+      assert.strictEqual(tokens[2].type, TokenType.SLASH);
+      assert.strictEqual(tokens[3].type, TokenType.NAME);
+      assert.strictEqual(tokens[3].value, "bar");
+    });
+
+    it("should tokenize descendant path", () => {
+      const tokens = tokenize("//foo");
+      assert.strictEqual(tokens[0].type, TokenType.DOUBLE_SLASH);
+      assert.strictEqual(tokens[1].type, TokenType.NAME);
+      assert.strictEqual(tokens[1].value, "foo");
+    });
+  });
+
+  describe("Wildcards and special characters", () => {
+    it("should tokenize wildcard", () => {
+      const tokens = tokenize("*");
+      assert.strictEqual(tokens[0].type, TokenType.STAR);
+    });
+
+    it("should tokenize attribute selector", () => {
+      const tokens = tokenize("@id");
+      assert.strictEqual(tokens[0].type, TokenType.AT);
+      assert.strictEqual(tokens[1].type, TokenType.NAME);
+      assert.strictEqual(tokens[1].value, "id");
+    });
+
+    it("should tokenize dot (self)", () => {
+      const tokens = tokenize(".");
+      assert.strictEqual(tokens[0].type, TokenType.DOT);
+    });
+
+    it("should tokenize double dot (parent)", () => {
+      const tokens = tokenize("..");
+      assert.strictEqual(tokens[0].type, TokenType.DOUBLE_DOT);
+    });
+  });
+
+  describe("Predicates", () => {
+    it("should tokenize predicate brackets", () => {
+      const tokens = tokenize("foo[1]");
+      assert.strictEqual(tokens[0].type, TokenType.NAME);
+      assert.strictEqual(tokens[1].type, TokenType.LBRACKET);
+      assert.strictEqual(tokens[2].type, TokenType.NUMBER);
+      assert.strictEqual(tokens[2].value, 1);
+      assert.strictEqual(tokens[3].type, TokenType.RBRACKET);
+    });
+
+    it("should tokenize complex predicate", () => {
+      const tokens = tokenize('item[@id="test"]');
+      assert.strictEqual(tokens[0].type, TokenType.NAME);
+      assert.strictEqual(tokens[1].type, TokenType.LBRACKET);
+      assert.strictEqual(tokens[2].type, TokenType.AT);
+      assert.strictEqual(tokens[3].type, TokenType.NAME);
+      assert.strictEqual(tokens[4].type, TokenType.EQUALS);
+      assert.strictEqual(tokens[5].type, TokenType.LITERAL);
+      assert.strictEqual(tokens[5].value, "test");
+      assert.strictEqual(tokens[6].type, TokenType.RBRACKET);
+    });
+  });
+
+  describe("Operators", () => {
+    it("should tokenize arithmetic operators", () => {
+      const tokens = tokenize("1 + 2 - 3 * 4");
+      assert.strictEqual(tokens[0].type, TokenType.NUMBER);
+      assert.strictEqual(tokens[1].type, TokenType.PLUS);
+      assert.strictEqual(tokens[2].type, TokenType.NUMBER);
+      assert.strictEqual(tokens[3].type, TokenType.MINUS);
+      assert.strictEqual(tokens[4].type, TokenType.NUMBER);
+      assert.strictEqual(tokens[5].type, TokenType.STAR);
+      assert.strictEqual(tokens[6].type, TokenType.NUMBER);
+    });
+
+    it("should tokenize comparison operators", () => {
+      const tokens = tokenize("a = b != c < d <= e > f >= g");
+      assert.strictEqual(tokens[1].type, TokenType.EQUALS);
+      assert.strictEqual(tokens[3].type, TokenType.NOT_EQUALS);
+      assert.strictEqual(tokens[5].type, TokenType.LT);
+      assert.strictEqual(tokens[7].type, TokenType.LTE);
+      assert.strictEqual(tokens[9].type, TokenType.GT);
+      assert.strictEqual(tokens[11].type, TokenType.GTE);
+    });
+
+    it("should tokenize logical operators", () => {
+      const tokens = tokenize("a and b or c");
+      assert.strictEqual(tokens[0].type, TokenType.NAME);
+      assert.strictEqual(tokens[1].type, TokenType.AND);
+      assert.strictEqual(tokens[2].type, TokenType.NAME);
+      assert.strictEqual(tokens[3].type, TokenType.OR);
+      assert.strictEqual(tokens[4].type, TokenType.NAME);
+    });
+
+    it("should tokenize div and mod", () => {
+      const tokens = tokenize("10 div 3 mod 2");
+      assert.strictEqual(tokens[1].type, TokenType.DIV);
+      assert.strictEqual(tokens[3].type, TokenType.MOD);
+    });
+
+    it("should tokenize union operator", () => {
+      const tokens = tokenize("a | b");
+      assert.strictEqual(tokens[1].type, TokenType.PIPE);
+    });
+  });
+
+  describe("Literals", () => {
+    it("should tokenize double-quoted string", () => {
+      const tokens = tokenize('"hello world"');
+      assert.strictEqual(tokens[0].type, TokenType.LITERAL);
+      assert.strictEqual(tokens[0].value, "hello world");
+    });
+
+    it("should tokenize single-quoted string", () => {
+      const tokens = tokenize("'hello world'");
+      assert.strictEqual(tokens[0].type, TokenType.LITERAL);
+      assert.strictEqual(tokens[0].value, "hello world");
+    });
+
+    it("should tokenize integer", () => {
+      const tokens = tokenize("42");
+      assert.strictEqual(tokens[0].type, TokenType.NUMBER);
+      assert.strictEqual(tokens[0].value, 42);
+    });
+
+    it("should tokenize decimal number", () => {
+      const tokens = tokenize("3.14");
+      assert.strictEqual(tokens[0].type, TokenType.NUMBER);
+      assert.strictEqual(tokens[0].value, 3.14);
+    });
+
+    it("should tokenize decimal starting with dot", () => {
+      const tokens = tokenize(".5");
+      assert.strictEqual(tokens[0].type, TokenType.NUMBER);
+      assert.strictEqual(tokens[0].value, 0.5);
+    });
+  });
+
+  describe("Functions", () => {
+    it("should tokenize function call", () => {
+      const tokens = tokenize("contains(a, b)");
+      assert.strictEqual(tokens[0].type, TokenType.FUNCTION);
+      assert.strictEqual(tokens[0].value, "contains");
+      assert.strictEqual(tokens[1].type, TokenType.LPAREN);
+      assert.strictEqual(tokens[2].type, TokenType.NAME);
+      assert.strictEqual(tokens[3].type, TokenType.COMMA);
+      assert.strictEqual(tokens[4].type, TokenType.NAME);
+      assert.strictEqual(tokens[5].type, TokenType.RPAREN);
+    });
+
+    it("should tokenize node type test", () => {
+      const tokens = tokenize("text()");
+      assert.strictEqual(tokens[0].type, TokenType.NODE_TYPE);
+      assert.strictEqual(tokens[0].value, "text");
+    });
+
+    it("should tokenize processing-instruction test", () => {
+      const tokens = tokenize("processing-instruction()");
+      assert.strictEqual(tokens[0].type, TokenType.NODE_TYPE);
+      assert.strictEqual(tokens[0].value, "processing-instruction");
+    });
+  });
+
+  describe("Axes", () => {
+    it("should tokenize axis with double colon", () => {
+      const tokens = tokenize("child::foo");
+      assert.strictEqual(tokens[0].type, TokenType.AXIS);
+      assert.strictEqual(tokens[0].value, "child");
+      assert.strictEqual(tokens[1].type, TokenType.DOUBLE_COLON);
+      assert.strictEqual(tokens[2].type, TokenType.NAME);
+    });
+
+    it("should tokenize descendant-or-self axis", () => {
+      const tokens = tokenize("descendant-or-self::node()");
+      assert.strictEqual(tokens[0].type, TokenType.AXIS);
+      assert.strictEqual(tokens[0].value, "descendant-or-self");
+    });
+  });
+
+  describe("Variables", () => {
+    it("should tokenize variable reference", () => {
+      const tokens = tokenize("$myVar");
+      assert.strictEqual(tokens[0].type, TokenType.DOLLAR);
+      assert.strictEqual(tokens[1].type, TokenType.NAME);
+      assert.strictEqual(tokens[1].value, "myVar");
+    });
+  });
+
+  describe("Complex expressions", () => {
+    it("should tokenize complex XPath expression", () => {
+      const expr = '//div[@class="content"]/p[position() > 1 and @id]';
+      const tokens = tokenize(expr);
+
+      // Verify tokenization completed without error
+      assert.ok(tokens.length > 0);
+      assert.strictEqual(tokens[tokens.length - 1].type, TokenType.EOF);
+    });
+  });
+});