@aigne/ash 0.0.1 → 0.0.2-beta.1

package/src/lexer.ts

@@ -1,222 +0,0 @@
-export type TokenType =
-  | "JOB" | "FIND" | "WHERE" | "MAP" | "SAVE" | "PUBLISH" | "TEE" | "FANOUT" | "INPUT" | "OUTPUT"
-  | "ACTION" | "ROUTE" | "LOOKUP" | "ARROW"
-  | "PLUS" | "MINUS" | "STAR" | "SLASH"
-  | "PIPE" | "ASSIGN" | "DOT" | "COMMA" | "COLON"
-  | "LPAREN" | "RPAREN" | "LBRACKET" | "RBRACKET" | "LBRACE" | "RBRACE"
-  | "IDENTIFIER" | "STRING" | "NUMBER" | "PATH" | "AT"
-  | "GT" | "LT" | "GTE" | "LTE" | "EQ" | "NEQ"
-  | "LET" | "PARAM" | "ON" | "DOLLAR" | "COUNT" | "GROUP_BY"
-  | "NEWLINE" | "EOF";
-
-export interface Token {
-  type: TokenType;
-  value: string;
-  line: number;
-  column: number;
-}
-
-const KEYWORDS: Record<string, TokenType> = {
-  job: "JOB",
-  find: "FIND",
-  where: "WHERE",
-  map: "MAP",
-  save: "SAVE",
-  publish: "PUBLISH",
-  tee: "TEE",
-  fanout: "FANOUT",
-  input: "INPUT",
-  output: "OUTPUT",
-  let: "LET",
-  count: "COUNT",
-  action: "ACTION",
-  route: "ROUTE",
-  lookup: "LOOKUP",
-  param: "PARAM",
-  on: "ON",
-};
-
-export class AshLexer {
-  tokenize(source: string): Token[] {
-    const tokens: Token[] = [];
-    let pos = 0;
-    let line = 1;
-    let col = 1;
-
-    while (pos < source.length) {
-      const ch = source[pos];
-
-      // Newline
-      if (ch === "\n" || ch === "\r") {
-        if (ch === "\r" && source[pos + 1] === "\n") pos++;
-        tokens.push({ type: "NEWLINE", value: "\\n", line, column: col });
-        pos++;
-        line++;
-        col = 1;
-        continue;
-      }
-
-      // Whitespace (skip)
-      if (ch === " " || ch === "\t") {
-        pos++;
-        col++;
-        continue;
-      }
-
-      // Comment: # to end of line
-      if (ch === "#") {
-        while (pos < source.length && source[pos] !== "\n" && source[pos] !== "\r") {
-          pos++;
-          col++;
-        }
-        continue;
-      }
-
-      // String literal
-      if (ch === '"') {
-        const startCol = col;
-        pos++; col++;
-        let value = "";
-        while (pos < source.length && source[pos] !== '"') {
-          if (source[pos] === "\\") {
-            pos++; col++;
-            if (pos >= source.length) break;
-            const escaped = source[pos];
-            if (escaped === '"') value += '"';
-            else if (escaped === "\\") value += "\\";
-            else if (escaped === "n") value += "\n";
-            else if (escaped === "t") value += "\t";
-            else if (escaped === "$") value += "\\$"; // preserve for template escape detection
-            else value += escaped;
-          } else {
-            if (source[pos] === "\n") {
-              throw new Error(`Unterminated string at line ${line}, column ${startCol}`);
-            }
-            value += source[pos];
-          }
-          pos++; col++;
-        }
-        if (pos >= source.length) {
-          throw new Error(`Unterminated string at line ${line}, column ${startCol}`);
-        }
-        pos++; col++; // closing quote
-        tokens.push({ type: "STRING", value, line, column: startCol });
-        continue;
-      }
-
-      // Number
-      if (ch >= "0" && ch <= "9") {
-        const startCol = col;
-        let num = "";
-        while (pos < source.length && ((source[pos] >= "0" && source[pos] <= "9") || source[pos] === ".")) {
-          num += source[pos];
-          pos++; col++;
-        }
-        tokens.push({ type: "NUMBER", value: num, line, column: startCol });
-        continue;
-      }
-
-      // Path or SLASH: `/` followed by a letter, `.`, `_`, or `/` starts a PATH; otherwise SLASH
-      if (ch === "/") {
-        const next = source[pos + 1];
-        if (next !== undefined && ((next >= "a" && next <= "z") || (next >= "A" && next <= "Z") || next === "_" || next === "." || next === "/")) {
-          const startCol = col;
-          let path = "";
-          while (pos < source.length && source[pos] !== " " && source[pos] !== "\t" && source[pos] !== "\n" && source[pos] !== "\r" && source[pos] !== "|" && source[pos] !== "}" && source[pos] !== ")" && source[pos] !== "," && source[pos] !== ":") {
-            // Handle ${...} template blocks atomically — don't stop at } inside template
-            if (source[pos] === "$" && pos + 1 < source.length && source[pos + 1] === "{") {
-              path += source[pos]; pos++; col++; // $
-              path += source[pos]; pos++; col++; // {
-              while (pos < source.length && source[pos] !== "}") {
-                path += source[pos]; pos++; col++;
-              }
-              if (pos < source.length) {
-                path += source[pos]; pos++; col++; // closing }
-              }
-              continue;
-            }
-            path += source[pos];
-            pos++; col++;
-          }
-          tokens.push({ type: "PATH", value: path, line, column: startCol });
-          continue;
-        }
-        tokens.push({ type: "SLASH", value: "/", line, column: col }); pos++; col++; continue;
-      }
-
-      // Identifier / keyword
-      if ((ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z") || ch === "_") {
-        const startCol = col;
-        let ident = "";
-        while (pos < source.length && ((source[pos] >= "a" && source[pos] <= "z") || (source[pos] >= "A" && source[pos] <= "Z") || (source[pos] >= "0" && source[pos] <= "9") || source[pos] === "_")) {
-          ident += source[pos];
-          pos++; col++;
-        }
-        // Handle compound keyword: group-by
-        if (ident === "group" && pos < source.length && source[pos] === "-") {
-          const rest = source.slice(pos + 1, pos + 3);
-          if (rest === "by") {
-            pos += 3; col += 3;
-            tokens.push({ type: "GROUP_BY", value: "group-by", line, column: startCol });
-            continue;
-          }
-        }
-        const kw = Object.hasOwn(KEYWORDS, ident) ? KEYWORDS[ident] : undefined;
-        tokens.push({ type: kw ?? "IDENTIFIER", value: ident, line, column: startCol });
-        continue;
-      }
-
-      // Operators and punctuation
-      // Arithmetic operators
-      if (ch === "+") { tokens.push({ type: "PLUS", value: "+", line, column: col }); pos++; col++; continue; }
-      if (ch === "-") {
-        if (source[pos + 1] === ">") {
-          tokens.push({ type: "ARROW", value: "->", line, column: col }); pos += 2; col += 2; continue;
-        }
-        tokens.push({ type: "MINUS", value: "-", line, column: col }); pos++; col++; continue;
-      }
-      if (ch === "*") { tokens.push({ type: "STAR", value: "*", line, column: col }); pos++; col++; continue; }
-
-      if (ch === "|") { tokens.push({ type: "PIPE", value: "|", line, column: col }); pos++; col++; continue; }
-      if (ch === "=") {
-        if (source[pos + 1] === "=") {
-          tokens.push({ type: "EQ", value: "==", line, column: col }); pos += 2; col += 2; continue;
-        }
-        tokens.push({ type: "ASSIGN", value: "=", line, column: col }); pos++; col++; continue;
-      }
-      if (ch === "!") {
-        if (source[pos + 1] === "=") {
-          tokens.push({ type: "NEQ", value: "!=", line, column: col }); pos += 2; col += 2; continue;
-        }
-      }
-      if (ch === ">") {
-        if (source[pos + 1] === "=") {
-          tokens.push({ type: "GTE", value: ">=", line, column: col }); pos += 2; col += 2; continue;
-        }
-        tokens.push({ type: "GT", value: ">", line, column: col }); pos++; col++; continue;
-      }
-      if (ch === "<") {
-        if (source[pos + 1] === "=") {
-          tokens.push({ type: "LTE", value: "<=", line, column: col }); pos += 2; col += 2; continue;
-        }
-        tokens.push({ type: "LT", value: "<", line, column: col }); pos++; col++; continue;
-      }
-      if (ch === ".") { tokens.push({ type: "DOT", value: ".", line, column: col }); pos++; col++; continue; }
-      if (ch === ",") { tokens.push({ type: "COMMA", value: ",", line, column: col }); pos++; col++; continue; }
-      if (ch === "(") { tokens.push({ type: "LPAREN", value: "(", line, column: col }); pos++; col++; continue; }
-      if (ch === ")") { tokens.push({ type: "RPAREN", value: ")", line, column: col }); pos++; col++; continue; }
-      if (ch === "[") { tokens.push({ type: "LBRACKET", value: "[", line, column: col }); pos++; col++; continue; }
-      if (ch === "]") { tokens.push({ type: "RBRACKET", value: "]", line, column: col }); pos++; col++; continue; }
-      if (ch === "{") { tokens.push({ type: "LBRACE", value: "{", line, column: col }); pos++; col++; continue; }
-      if (ch === "}") { tokens.push({ type: "RBRACE", value: "}", line, column: col }); pos++; col++; continue; }
-      if (ch === "@") { tokens.push({ type: "AT", value: "@", line, column: col }); pos++; col++; continue; }
-      if (ch === "$") { tokens.push({ type: "DOLLAR", value: "$", line, column: col }); pos++; col++; continue; }
-      if (ch === ":") { tokens.push({ type: "COLON", value: ":", line, column: col }); pos++; col++; continue; }
-
-      throw new Error(`Unrecognized character '${ch}' at line ${line}, column ${col}`);
-    }
-
-    tokens.push({ type: "EOF", value: "", line, column: col });
-    return tokens;
-  }
-}