@aigne/ash 0.0.1

package/src/lexer.ts

@@ -0,0 +1,222 @@
+export type TokenType =
+  | "JOB" | "FIND" | "WHERE" | "MAP" | "SAVE" | "PUBLISH" | "TEE" | "FANOUT" | "INPUT" | "OUTPUT"
+  | "ACTION" | "ROUTE" | "LOOKUP" | "ARROW"
+  | "PLUS" | "MINUS" | "STAR" | "SLASH"
+  | "PIPE" | "ASSIGN" | "DOT" | "COMMA" | "COLON"
+  | "LPAREN" | "RPAREN" | "LBRACKET" | "RBRACKET" | "LBRACE" | "RBRACE"
+  | "IDENTIFIER" | "STRING" | "NUMBER" | "PATH" | "AT"
+  | "GT" | "LT" | "GTE" | "LTE" | "EQ" | "NEQ"
+  | "LET" | "PARAM" | "ON" | "DOLLAR" | "COUNT" | "GROUP_BY"
+  | "NEWLINE" | "EOF";
+
+export interface Token {
+  type: TokenType;
+  value: string;
+  line: number;
+  column: number;
+}
+
+const KEYWORDS: Record<string, TokenType> = {
+  job: "JOB",
+  find: "FIND",
+  where: "WHERE",
+  map: "MAP",
+  save: "SAVE",
+  publish: "PUBLISH",
+  tee: "TEE",
+  fanout: "FANOUT",
+  input: "INPUT",
+  output: "OUTPUT",
+  let: "LET",
+  count: "COUNT",
+  action: "ACTION",
+  route: "ROUTE",
+  lookup: "LOOKUP",
+  param: "PARAM",
+  on: "ON",
+};
+
+export class AshLexer {
+  tokenize(source: string): Token[] {
+    const tokens: Token[] = [];
+    let pos = 0;
+    let line = 1;
+    let col = 1;
+
+    while (pos < source.length) {
+      const ch = source[pos];
+
+      // Newline
+      if (ch === "\n" || ch === "\r") {
+        if (ch === "\r" && source[pos + 1] === "\n") pos++;
+        tokens.push({ type: "NEWLINE", value: "\\n", line, column: col });
+        pos++;
+        line++;
+        col = 1;
+        continue;
+      }
+
+      // Whitespace (skip)
+      if (ch === " " || ch === "\t") {
+        pos++;
+        col++;
+        continue;
+      }
+
+      // Comment: # to end of line
+      if (ch === "#") {
+        while (pos < source.length && source[pos] !== "\n" && source[pos] !== "\r") {
+          pos++;
+          col++;
+        }
+        continue;
+      }
+
+      // String literal
+      if (ch === '"') {
+        const startCol = col;
+        pos++; col++;
+        let value = "";
+        while (pos < source.length && source[pos] !== '"') {
+          if (source[pos] === "\\") {
+            pos++; col++;
+            if (pos >= source.length) break;
+            const escaped = source[pos];
+            if (escaped === '"') value += '"';
+            else if (escaped === "\\") value += "\\";
+            else if (escaped === "n") value += "\n";
+            else if (escaped === "t") value += "\t";
+            else if (escaped === "$") value += "\\$"; // preserve for template escape detection
+            else value += escaped;
+          } else {
+            if (source[pos] === "\n") {
+              throw new Error(`Unterminated string at line ${line}, column ${startCol}`);
+            }
+            value += source[pos];
+          }
+          pos++; col++;
+        }
+        if (pos >= source.length) {
+          throw new Error(`Unterminated string at line ${line}, column ${startCol}`);
+        }
+        pos++; col++; // closing quote
+        tokens.push({ type: "STRING", value, line, column: startCol });
+        continue;
+      }
+
+      // Number
+      if (ch >= "0" && ch <= "9") {
+        const startCol = col;
+        let num = "";
+        while (pos < source.length && ((source[pos] >= "0" && source[pos] <= "9") || source[pos] === ".")) {
+          num += source[pos];
+          pos++; col++;
+        }
+        tokens.push({ type: "NUMBER", value: num, line, column: startCol });
+        continue;
+      }
+
+      // Path or SLASH: `/` followed by a letter, `.`, `_`, or `/` starts a PATH; otherwise SLASH
+      if (ch === "/") {
+        const next = source[pos + 1];
+        if (next !== undefined && ((next >= "a" && next <= "z") || (next >= "A" && next <= "Z") || next === "_" || next === "." || next === "/")) {
+          const startCol = col;
+          let path = "";
+          while (pos < source.length && source[pos] !== " " && source[pos] !== "\t" && source[pos] !== "\n" && source[pos] !== "\r" && source[pos] !== "|" && source[pos] !== "}" && source[pos] !== ")" && source[pos] !== "," && source[pos] !== ":") {
+            // Handle ${...} template blocks atomically — don't stop at } inside template
+            if (source[pos] === "$" && pos + 1 < source.length && source[pos + 1] === "{") {
+              path += source[pos]; pos++; col++; // $
+              path += source[pos]; pos++; col++; // {
+              while (pos < source.length && source[pos] !== "}") {
+                path += source[pos]; pos++; col++;
+              }
+              if (pos < source.length) {
+                path += source[pos]; pos++; col++; // closing }
+              }
+              continue;
+            }
+            path += source[pos];
+            pos++; col++;
+          }
+          tokens.push({ type: "PATH", value: path, line, column: startCol });
+          continue;
+        }
+        tokens.push({ type: "SLASH", value: "/", line, column: col }); pos++; col++; continue;
+      }
+
+      // Identifier / keyword
+      if ((ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z") || ch === "_") {
+        const startCol = col;
+        let ident = "";
+        while (pos < source.length && ((source[pos] >= "a" && source[pos] <= "z") || (source[pos] >= "A" && source[pos] <= "Z") || (source[pos] >= "0" && source[pos] <= "9") || source[pos] === "_")) {
+          ident += source[pos];
+          pos++; col++;
+        }
+        // Handle compound keyword: group-by
+        if (ident === "group" && pos < source.length && source[pos] === "-") {
+          const rest = source.slice(pos + 1, pos + 3);
+          if (rest === "by") {
+            pos += 3; col += 3;
+            tokens.push({ type: "GROUP_BY", value: "group-by", line, column: startCol });
+            continue;
+          }
+        }
+        const kw = Object.hasOwn(KEYWORDS, ident) ? KEYWORDS[ident] : undefined;
+        tokens.push({ type: kw ?? "IDENTIFIER", value: ident, line, column: startCol });
+        continue;
+      }
+
+      // Operators and punctuation
+      // Arithmetic operators
+      if (ch === "+") { tokens.push({ type: "PLUS", value: "+", line, column: col }); pos++; col++; continue; }
+      if (ch === "-") {
+        if (source[pos + 1] === ">") {
+          tokens.push({ type: "ARROW", value: "->", line, column: col }); pos += 2; col += 2; continue;
+        }
+        tokens.push({ type: "MINUS", value: "-", line, column: col }); pos++; col++; continue;
+      }
+      if (ch === "*") { tokens.push({ type: "STAR", value: "*", line, column: col }); pos++; col++; continue; }
+
+      if (ch === "|") { tokens.push({ type: "PIPE", value: "|", line, column: col }); pos++; col++; continue; }
+      if (ch === "=") {
+        if (source[pos + 1] === "=") {
+          tokens.push({ type: "EQ", value: "==", line, column: col }); pos += 2; col += 2; continue;
+        }
+        tokens.push({ type: "ASSIGN", value: "=", line, column: col }); pos++; col++; continue;
+      }
+      if (ch === "!") {
+        if (source[pos + 1] === "=") {
+          tokens.push({ type: "NEQ", value: "!=", line, column: col }); pos += 2; col += 2; continue;
+        }
+      }
+      if (ch === ">") {
+        if (source[pos + 1] === "=") {
+          tokens.push({ type: "GTE", value: ">=", line, column: col }); pos += 2; col += 2; continue;
+        }
+        tokens.push({ type: "GT", value: ">", line, column: col }); pos++; col++; continue;
+      }
+      if (ch === "<") {
+        if (source[pos + 1] === "=") {
+          tokens.push({ type: "LTE", value: "<=", line, column: col }); pos += 2; col += 2; continue;
+        }
+        tokens.push({ type: "LT", value: "<", line, column: col }); pos++; col++; continue;
+      }
+      if (ch === ".") { tokens.push({ type: "DOT", value: ".", line, column: col }); pos++; col++; continue; }
+      if (ch === ",") { tokens.push({ type: "COMMA", value: ",", line, column: col }); pos++; col++; continue; }
+      if (ch === "(") { tokens.push({ type: "LPAREN", value: "(", line, column: col }); pos++; col++; continue; }
+      if (ch === ")") { tokens.push({ type: "RPAREN", value: ")", line, column: col }); pos++; col++; continue; }
+      if (ch === "[") { tokens.push({ type: "LBRACKET", value: "[", line, column: col }); pos++; col++; continue; }
+      if (ch === "]") { tokens.push({ type: "RBRACKET", value: "]", line, column: col }); pos++; col++; continue; }
+      if (ch === "{") { tokens.push({ type: "LBRACE", value: "{", line, column: col }); pos++; col++; continue; }
+      if (ch === "}") { tokens.push({ type: "RBRACE", value: "}", line, column: col }); pos++; col++; continue; }
+      if (ch === "@") { tokens.push({ type: "AT", value: "@", line, column: col }); pos++; col++; continue; }
+      if (ch === "$") { tokens.push({ type: "DOLLAR", value: "$", line, column: col }); pos++; col++; continue; }
+      if (ch === ":") { tokens.push({ type: "COLON", value: ":", line, column: col }); pos++; col++; continue; }
+
+      throw new Error(`Unrecognized character '${ch}' at line ${line}, column ${col}`);
+    }
+
+    tokens.push({ type: "EOF", value: "", line, column: col });
+    return tokens;
+  }
+}