@stacksjs/stx 0.2.95 → 0.2.97

package/dist/parser/tokenizer.js

@@ -1,2 +1,534 @@
-// @bun
-import{ey as a,fy as b,gy as c,hy as d}from"../chunk-bs3gmfxz.js";import"../chunk-xv34zbp4.js";export{d as splitByPipe,b as findMatchingDelimiter,c as extractBalancedExpression,a as Tokenizer};
+export class Tokenizer {
+  source;
+  pos = 0;
+  line = 1;
+  column = 0;
+  tokens = [];
+  constructor(source) {
+    this.source = source;
+  }
+  tokenize() {
+    this.tokens = [];
+    this.pos = 0;
+    this.line = 1;
+    this.column = 0;
+    while (this.pos < this.source.length) {
+      const token = this.nextToken();
+      if (token)
+        this.tokens.push(token);
+    }
+    this.tokens.push({
+      type: "EOF",
+      value: "",
+      start: this.pos,
+      end: this.pos,
+      line: this.line,
+      column: this.column
+    });
+    return this.tokens;
+  }
+  nextToken() {
+    if (this.pos >= this.source.length)
+      return null;
+    const start = this.pos, startLine = this.line, startColumn = this.column, char = this.source[this.pos];
+    if (/\s/.test(char))
+      return this.readWhitespace(start, startLine, startColumn);
+    if (char === "/" && this.peek(1) === "/")
+      return this.readLineComment(start, startLine, startColumn);
+    if (char === "/" && this.peek(1) === "*")
+      return this.readBlockComment(start, startLine, startColumn);
+    if (char === '"' || char === "'")
+      return this.readString(char, start, startLine, startColumn);
+    if (char === "`")
+      return this.readTemplateString(start, startLine, startColumn);
+    if (/\d/.test(char) || char === "." && /\d/.test(this.peek(1) || ""))
+      return this.readNumber(start, startLine, startColumn);
+    if (/[a-z_$]/i.test(char))
+      return this.readIdentifier(start, startLine, startColumn);
+    if (char === "{") {
+      if (this.peek(1) === "{" && this.peek(2) === "{") {
+        this.advance(3);
+        return { type: "RAW_START", value: "{{{", start, end: this.pos, line: startLine, column: startColumn };
+      }
+      if (this.peek(1) === "{") {
+        this.advance(2);
+        return { type: "EXPRESSION_START", value: "{{", start, end: this.pos, line: startLine, column: startColumn };
+      }
+      if (this.peek(1) === "!" && this.peek(2) === "!") {
+        this.advance(3);
+        return { type: "RAW_START", value: "{!!", start, end: this.pos, line: startLine, column: startColumn };
+      }
+    }
+    if (char === "}") {
+      if (this.peek(1) === "}" && this.peek(2) === "}") {
+        this.advance(3);
+        return { type: "RAW_END", value: "}}}", start, end: this.pos, line: startLine, column: startColumn };
+      }
+      if (this.peek(1) === "}") {
+        this.advance(2);
+        return { type: "EXPRESSION_END", value: "}}", start, end: this.pos, line: startLine, column: startColumn };
+      }
+    }
+    if (char === "!" && this.peek(1) === "!" && this.peek(2) === "}") {
+      this.advance(3);
+      return { type: "RAW_END", value: "!!}", start, end: this.pos, line: startLine, column: startColumn };
+    }
+    if (char === "@" && /[a-z]/i.test(this.peek(1) || ""))
+      return this.readDirective(start, startLine, startColumn);
+    const operators = [
+      "===",
+      "!==",
+      "==",
+      "!=",
+      "<=",
+      ">=",
+      "&&",
+      "||",
+      "??",
+      "++",
+      "--",
+      "+=",
+      "-=",
+      "*=",
+      "/=",
+      "%=",
+      "**",
+      "=>",
+      "?.",
+      "...",
+      "<<",
+      ">>",
+      ">>>",
+      "&=",
+      "|=",
+      "^="
+    ];
+    for (const op of operators)
+      if (this.source.slice(this.pos, this.pos + op.length) === op) {
+        this.advance(op.length);
+        return { type: "OPERATOR", value: op, start, end: this.pos, line: startLine, column: startColumn };
+      }
+    if (/[+\-*/%<>=!&|^~?:]/.test(char)) {
+      this.advance();
+      return { type: "OPERATOR", value: char, start, end: this.pos, line: startLine, column: startColumn };
+    }
+    if (/[()[\]{},;.]/.test(char)) {
+      this.advance();
+      return { type: "PUNCTUATION", value: char, start, end: this.pos, line: startLine, column: startColumn };
+    }
+    this.advance();
+    return { type: "PUNCTUATION", value: char, start, end: this.pos, line: startLine, column: startColumn };
+  }
+  readWhitespace(start, startLine, startColumn) {
+    while (this.pos < this.source.length && /\s/.test(this.source[this.pos]))
+      this.advance();
+    return {
+      type: "WHITESPACE",
+      value: this.source.slice(start, this.pos),
+      start,
+      end: this.pos,
+      line: startLine,
+      column: startColumn
+    };
+  }
+  readLineComment(start, startLine, startColumn) {
+    this.advance(2);
+    while (this.pos < this.source.length && this.source[this.pos] !== `
+`)
+      this.advance();
+    return {
+      type: "COMMENT",
+      value: this.source.slice(start, this.pos),
+      start,
+      end: this.pos,
+      line: startLine,
+      column: startColumn
+    };
+  }
+  readBlockComment(start, startLine, startColumn) {
+    this.advance(2);
+    while (this.pos < this.source.length - 1) {
+      if (this.source[this.pos] === "*" && this.source[this.pos + 1] === "/") {
+        this.advance(2);
+        break;
+      }
+      this.advance();
+    }
+    return {
+      type: "COMMENT",
+      value: this.source.slice(start, this.pos),
+      start,
+      end: this.pos,
+      line: startLine,
+      column: startColumn
+    };
+  }
+  readString(quote, start, startLine, startColumn) {
+    this.advance();
+    while (this.pos < this.source.length) {
+      const char = this.source[this.pos];
+      if (char === "\\") {
+        this.advance(2);
+        continue;
+      }
+      if (char === quote) {
+        this.advance();
+        break;
+      }
+      this.advance();
+    }
+    return {
+      type: "STRING",
+      value: this.source.slice(start, this.pos),
+      start,
+      end: this.pos,
+      line: startLine,
+      column: startColumn
+    };
+  }
+  readTemplateString(start, startLine, startColumn) {
+    this.advance();
+    let depth = 0;
+    const MAX_TEMPLATE_NESTING = 256;
+    while (this.pos < this.source.length) {
+      const char = this.source[this.pos];
+      if (char === "\\") {
+        this.advance(2);
+        continue;
+      }
+      if (char === "$" && this.peek(1) === "{") {
+        if (depth >= MAX_TEMPLATE_NESTING) {
+          this.pos = this.source.length;
+          break;
+        }
+        this.advance(2);
+        depth++;
+        continue;
+      }
+      if (char === "{" && depth > 0) {
+        depth++;
+        this.advance();
+        continue;
+      }
+      if (char === "}" && depth > 0) {
+        depth--;
+        this.advance();
+        continue;
+      }
+      if (char === "`" && depth === 0) {
+        this.advance();
+        break;
+      }
+      this.advance();
+    }
+    return {
+      type: "TEMPLATE_STRING",
+      value: this.source.slice(start, this.pos),
+      start,
+      end: this.pos,
+      line: startLine,
+      column: startColumn
+    };
+  }
+  readNumber(start, startLine, startColumn) {
+    if (this.source[this.pos] === "0") {
+      const next = this.peek(1)?.toLowerCase();
+      if (next === "x" || next === "o" || next === "b") {
+        this.advance(2);
+        while (this.pos < this.source.length && /[\da-f_]/i.test(this.source[this.pos]))
+          this.advance();
+        return {
+          type: "NUMBER",
+          value: this.source.slice(start, this.pos),
+          start,
+          end: this.pos,
+          line: startLine,
+          column: startColumn
+        };
+      }
+    }
+    while (this.pos < this.source.length && /[\d_]/.test(this.source[this.pos]))
+      this.advance();
+    if (this.source[this.pos] === "." && /\d/.test(this.peek(1) || "")) {
+      this.advance();
+      while (this.pos < this.source.length && /[\d_]/.test(this.source[this.pos]))
+        this.advance();
+    }
+    if (/e/i.test(this.source[this.pos] || "")) {
+      this.advance();
+      if (/[+-]/.test(this.source[this.pos] || ""))
+        this.advance();
+      while (this.pos < this.source.length && /[\d_]/.test(this.source[this.pos]))
+        this.advance();
+    }
+    if (this.source[this.pos] === "n")
+      this.advance();
+    return {
+      type: "NUMBER",
+      value: this.source.slice(start, this.pos),
+      start,
+      end: this.pos,
+      line: startLine,
+      column: startColumn
+    };
+  }
+  readIdentifier(start, startLine, startColumn) {
+    while (this.pos < this.source.length && /[\w$]/.test(this.source[this.pos]))
+      this.advance();
+    return {
+      type: "IDENTIFIER",
+      value: this.source.slice(start, this.pos),
+      start,
+      end: this.pos,
+      line: startLine,
+      column: startColumn
+    };
+  }
+  readDirective(start, startLine, startColumn) {
+    this.advance();
+    while (this.pos < this.source.length && /[a-z]/i.test(this.source[this.pos]))
+      this.advance();
+    return {
+      type: "DIRECTIVE",
+      value: this.source.slice(start, this.pos),
+      start,
+      end: this.pos,
+      line: startLine,
+      column: startColumn
+    };
+  }
+  advance(count = 1) {
+    for (let i = 0;i < count && this.pos < this.source.length; i++) {
+      if (this.source[this.pos] === `
+`) {
+        this.line++;
+        this.column = 0;
+      } else
+        this.column++;
+      this.pos++;
+    }
+  }
+  peek(offset) {
+    return this.source[this.pos + offset];
+  }
+  saveState() {
+    return { pos: this.pos, line: this.line, column: this.column };
+  }
+  restoreState(state) {
+    this.pos = state.pos;
+    this.line = state.line;
+    this.column = state.column;
+  }
+}
+export function findMatchingDelimiter(source, openChar, closeChar, startPos = 0) {
+  let depth = 0, pos = startPos, inString = null, inTemplateExpr = 0, isEscaped = !1;
+  while (pos < source.length) {
+    const char = source[pos];
+    if (inString && isEscaped) {
+      isEscaped = !1;
+      pos++;
+      continue;
+    }
+    if (inString && char === "\\") {
+      isEscaped = !0;
+      pos++;
+      continue;
+    }
+    isEscaped = !1;
+    if ((char === '"' || char === "'" || char === "`") && !inString) {
+      inString = char;
+      pos++;
+      continue;
+    }
+    if (char === inString && inString !== "`") {
+      inString = null;
+      pos++;
+      continue;
+    }
+    if (inString === "`" && char === "$" && source[pos + 1] === "{") {
+      inTemplateExpr++;
+      pos += 2;
+      continue;
+    }
+    if (inTemplateExpr > 0 && char === "}") {
+      inTemplateExpr--;
+      pos++;
+      continue;
+    }
+    if (char === "`" && inString === "`" && inTemplateExpr === 0) {
+      inString = null;
+      pos++;
+      continue;
+    }
+    if (inString) {
+      pos++;
+      continue;
+    }
+    if (char === openChar)
+      depth++;
+    else if (char === closeChar) {
+      depth--;
+      if (depth === 0)
+        return pos;
+    }
+    pos++;
+  }
+  return -1;
+}
+export function extractBalancedExpression(source, startPos) {
+  let pos = startPos;
+  const depth = { paren: 0, bracket: 0, brace: 0 };
+  let inString = null, inTemplateExpr = 0;
+  while (pos < source.length) {
+    const char = source[pos];
+    if ((pos > 0 ? source[pos - 1] : "") === "\\" && inString) {
+      pos++;
+      continue;
+    }
+    if ((char === '"' || char === "'") && !inString) {
+      inString = char;
+      pos++;
+      continue;
+    }
+    if (char === inString && inString !== "`") {
+      inString = null;
+      pos++;
+      continue;
+    }
+    if (char === "`" && !inString) {
+      inString = "`";
+      pos++;
+      continue;
+    }
+    if (char === "`" && inString === "`" && inTemplateExpr === 0) {
+      inString = null;
+      pos++;
+      continue;
+    }
+    if (inString === "`" && char === "$" && source[pos + 1] === "{") {
+      inTemplateExpr++;
+      pos += 2;
+      continue;
+    }
+    if (inTemplateExpr > 0) {
+      if (char === "{")
+        inTemplateExpr++;
+      else if (char === "}")
+        inTemplateExpr--;
+      pos++;
+      continue;
+    }
+    if (inString) {
+      pos++;
+      continue;
+    }
+    if (depth.paren === 0 && depth.bracket === 0 && depth.brace === 0) {
+      if (source.slice(pos, pos + 2) === "}}" || source.slice(pos, pos + 3) === "!!}")
+        break;
+      if (char === "," || char === ";")
+        break;
+    }
+    if (char === "(")
+      depth.paren++;
+    else if (char === ")") {
+      depth.paren--;
+      if (depth.paren < 0)
+        break;
+    } else if (char === "[")
+      depth.bracket++;
+    else if (char === "]") {
+      depth.bracket--;
+      if (depth.bracket < 0)
+        break;
+    } else if (char === "{")
+      depth.brace++;
+    else if (char === "}") {
+      depth.brace--;
+      if (depth.brace < 0)
+        break;
+    }
+    pos++;
+  }
+  return {
+    expression: source.slice(startPos, pos),
+    endPos: pos
+  };
+}
+export function splitByPipe(expression) {
+  const parts = [];
+  let current = "";
+  const depth = { paren: 0, bracket: 0, brace: 0 };
+  let inString = null, inTemplateExpr = 0;
+  for (let i = 0;i < expression.length; i++) {
+    const char = expression[i];
+    if ((i > 0 ? expression[i - 1] : "") === "\\" && inString) {
+      current += char;
+      continue;
+    }
+    if ((char === '"' || char === "'") && !inString) {
+      inString = char;
+      current += char;
+      continue;
+    }
+    if (char === inString && inString !== "`") {
+      inString = null;
+      current += char;
+      continue;
+    }
+    if (char === "`" && !inString) {
+      inString = "`";
+      current += char;
+      continue;
+    }
+    if (char === "`" && inString === "`" && inTemplateExpr === 0) {
+      inString = null;
+      current += char;
+      continue;
+    }
+    if (inString === "`" && char === "$" && expression[i + 1] === "{") {
+      inTemplateExpr++;
+      current += char + expression[i + 1];
+      i++;
+      continue;
+    }
+    if (inTemplateExpr > 0 && char === "{") {
+      inTemplateExpr++;
+      current += char;
+      continue;
+    }
+    if (inTemplateExpr > 0 && char === "}") {
+      inTemplateExpr--;
+      current += char;
+      continue;
+    }
+    if (inString) {
+      current += char;
+      continue;
+    }
+    if (char === "(")
+      depth.paren++;
+    else if (char === ")")
+      depth.paren--;
+    else if (char === "[")
+      depth.bracket++;
+    else if (char === "]")
+      depth.bracket--;
+    else if (char === "{")
+      depth.brace++;
+    else if (char === "}")
+      depth.brace--;
+    if (char === "|" && expression[i + 1] !== "|" && (i === 0 || expression[i - 1] !== "|") && expression[i - 1] !== "\\" && depth.paren === 0 && depth.bracket === 0 && depth.brace === 0) {
+      parts.push(current.trim());
+      current = "";
+      continue;
+    }
+    if (char === "|" && expression[i - 1] === "\\" && !inString) {
+      current = current.slice(0, -1) + char;
+      continue;
+    }
+    current += char;
+  }
+  if (current.trim())
+    parts.push(current.trim());
+  return parts;
+}