@aigne/ash 0.0.1

package/src/lexer.test.ts

@@ -0,0 +1,538 @@
+import { describe, it, expect } from "vitest";
+import { AshLexer } from "./lexer.js";
+import type { Token } from "./lexer.js";
+
+const lex = new AshLexer();
+
+function types(tokens: Token[]): string[] {
+  return tokens.map((t) => t.type);
+}
+
+describe("Phase 0: ASH Lexer", () => {
+  // ── Happy ──
+
+  it("tokenize single keyword `job` → JOB token", () => {
+    const tokens = lex.tokenize("job");
+    expect(tokens[0]).toMatchObject({ type: "JOB", value: "job" });
+  });
+
+  it("tokenize all keywords: find, where, map, save, publish, tee, fanout", () => {
+    const keywords = ["find", "where", "map", "save", "publish", "tee", "fanout"];
+    for (const kw of keywords) {
+      const tokens = lex.tokenize(kw);
+      expect(tokens[0].type).toBe(kw.toUpperCase());
+    }
+  });
+
+  it("tokenize pipe `|` → PIPE token", () => {
+    const tokens = lex.tokenize("|");
+    expect(tokens[0]).toMatchObject({ type: "PIPE", value: "|" });
+  });
+
+  it("tokenize assign `=` → ASSIGN token", () => {
+    const tokens = lex.tokenize("=");
+    expect(tokens[0]).toMatchObject({ type: "ASSIGN", value: "=" });
+  });
+
+  it("tokenize punctuation: . , () [] {}", () => {
+    const tokens = lex.tokenize(". , ( ) [ ] { }");
+    const t = types(tokens);
+    expect(t).toContain("DOT");
+    expect(t).toContain("COMMA");
+    expect(t).toContain("LPAREN");
+    expect(t).toContain("RPAREN");
+    expect(t).toContain("LBRACKET");
+    expect(t).toContain("RBRACKET");
+    expect(t).toContain("LBRACE");
+    expect(t).toContain("RBRACE");
+  });
+
+  it("tokenize string literal `\"hello world\"` with space", () => {
+    const tokens = lex.tokenize('"hello world"');
+    expect(tokens[0]).toMatchObject({ type: "STRING", value: "hello world" });
+  });
+
+  it("tokenize numbers: integer 42, float 3.14", () => {
+    expect(lex.tokenize("42")[0]).toMatchObject({ type: "NUMBER", value: "42" });
+    expect(lex.tokenize("3.14")[0]).toMatchObject({ type: "NUMBER", value: "3.14" });
+  });
+
+  it("tokenize path `/world/users/active` → PATH token", () => {
+    const tokens = lex.tokenize("/world/users/active");
+    expect(tokens[0]).toMatchObject({ type: "PATH", value: "/world/users/active" });
+  });
+
+  it("tokenize annotation `@` → AT token", () => {
+    const tokens = lex.tokenize("@");
+    expect(tokens[0]).toMatchObject({ type: "AT", value: "@" });
+  });
+
+  it("tokenize identifier `myVar123`", () => {
+    const tokens = lex.tokenize("myVar123");
+    expect(tokens[0]).toMatchObject({ type: "IDENTIFIER", value: "myVar123" });
+  });
+
+  it("tokenize full pipeline", () => {
+    const tokens = lex.tokenize("find /world/users | where age > 18 | save /world/results");
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["FIND", "PATH", "PIPE", "WHERE", "IDENTIFIER", "GT", "NUMBER", "PIPE", "SAVE", "PATH"]);
+  });
+
+  it("multiline source has correct line/column", () => {
+    const tokens = lex.tokenize("job\nfind");
+    const job = tokens[0];
+    expect(job.line).toBe(1);
+    const find = tokens.find((t) => t.type === "FIND")!;
+    expect(find.line).toBe(2);
+    expect(find.column).toBe(1);
+  });
+
+  it("NEWLINE token between statements", () => {
+    const tokens = lex.tokenize("job\nfind");
+    expect(types(tokens)).toContain("NEWLINE");
+  });
+
+  it("last token is always EOF", () => {
+    expect(lex.tokenize("job").at(-1)!.type).toBe("EOF");
+    expect(lex.tokenize("").at(-1)!.type).toBe("EOF");
+  });
+
+  // ── Bad ──
+
+  it("unterminated string → error with position", () => {
+    expect(() => lex.tokenize('"hello')).toThrow(/unterminated.*string/i);
+  });
+
+  it("`#` is a comment, not an error", () => {
+    const tokens = lex.tokenize("# this is a comment");
+    expect(types(tokens)).toEqual(["EOF"]);
+  });
+
+  // ── Edge ──
+
+  it("empty source → only EOF", () => {
+    const tokens = lex.tokenize("");
+    expect(tokens).toHaveLength(1);
+    expect(tokens[0].type).toBe("EOF");
+  });
+
+  it("whitespace only → NEWLINE(s) + EOF or just EOF", () => {
+    const tokens = lex.tokenize("  \t  ");
+    expect(tokens).toHaveLength(1); // just EOF, no newlines
+    expect(tokens[0].type).toBe("EOF");
+
+    const tokens2 = lex.tokenize("\n\n");
+    const t2 = types(tokens2);
+    expect(t2.filter((x) => x === "NEWLINE")).toHaveLength(2);
+    expect(t2.at(-1)).toBe("EOF");
+  });
+
+  it("string with escaped quote", () => {
+    const tokens = lex.tokenize('"say \\"hi\\""');
+    expect(tokens[0]).toMatchObject({ type: "STRING", value: 'say "hi"' });
+  });
+
+  it("bare `/` followed by space → SLASH token (not path)", () => {
+    const tokens = lex.tokenize("/ ");
+    expect(tokens[0]).toMatchObject({ type: "SLASH", value: "/" });
+  });
+
+  it("MAX_SAFE_INTEGER tokenized correctly", () => {
+    const n = String(Number.MAX_SAFE_INTEGER);
+    const tokens = lex.tokenize(n);
+    expect(tokens[0]).toMatchObject({ type: "NUMBER", value: n });
+  });
+
+  it("keyword prefix identifier `finder` → IDENTIFIER not FIND", () => {
+    const tokens = lex.tokenize("finder");
+    expect(tokens[0]).toMatchObject({ type: "IDENTIFIER", value: "finder" });
+  });
+
+  // ── Security ──
+
+  it("string containing ${process.exit()} → treated as string data", () => {
+    const tokens = lex.tokenize('"${process.exit()}"');
+    expect(tokens[0]).toMatchObject({ type: "STRING", value: "${process.exit()}" });
+  });
+
+  it("path with /../../../etc/passwd → treated as PATH token", () => {
+    const tokens = lex.tokenize("/../../../etc/passwd");
+    expect(tokens[0]).toMatchObject({ type: "PATH" });
+    expect(tokens[0].value).toContain("/../");
+  });
+
+  // ── Output / Input keywords ──
+
+  it("tokenize `input` → INPUT token", () => {
+    const tokens = lex.tokenize("input");
+    expect(tokens[0]).toMatchObject({ type: "INPUT", value: "input" });
+  });
+
+  it("tokenize output with string: `output \"hello\"` → OUTPUT STRING", () => {
+    const tokens = lex.tokenize('output "hello"');
+    expect(types(tokens).filter((x) => x !== "EOF")).toEqual(["OUTPUT", "STRING"]);
+  });
+
+  it("tokenize input with string: `input \"name?\"` → INPUT STRING", () => {
+    const tokens = lex.tokenize('input "name?"');
+    expect(types(tokens).filter((x) => x !== "EOF")).toEqual(["INPUT", "STRING"]);
+  });
+
+  it("keyword prefix `outputter` → IDENTIFIER not OUTPUT", () => {
+    const tokens = lex.tokenize("outputter");
+    expect(tokens[0]).toMatchObject({ type: "IDENTIFIER", value: "outputter" });
+  });
+
+  // ── Data Loss ──
+
+  it("mixed line endings \\r\\n and \\n → consistent line numbers", () => {
+    const tokens = lex.tokenize("job\r\nfind\nmap");
+    const find = tokens.find((t) => t.type === "FIND")!;
+    const map = tokens.find((t) => t.type === "MAP")!;
+    expect(find.line).toBe(2);
+    expect(map.line).toBe(3);
+  });
+});
+
+describe("Phase 5: Comments", () => {
+  // ── Happy ──
+
+  it("full line comment → skipped, no token produced", () => {
+    const tokens = lex.tokenize("# full line comment");
+    expect(types(tokens)).toEqual(["EOF"]);
+  });
+
+  it("inline comment after code → only code tokens produced", () => {
+    const tokens = lex.tokenize("find /world/x # inline comment");
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["FIND", "PATH"]);
+  });
+
+  it("line numbers correct after comment lines", () => {
+    const tokens = lex.tokenize("# comment\njob");
+    const job = tokens.find((t) => t.type === "JOB")!;
+    expect(job.line).toBe(2);
+  });
+
+  it("multiple consecutive comment lines → all skipped", () => {
+    const tokens = lex.tokenize("# line 1\n# line 2\n# line 3\njob");
+    const nonNewlineEof = types(tokens).filter((x) => x !== "NEWLINE" && x !== "EOF");
+    expect(nonNewlineEof).toEqual(["JOB"]);
+  });
+
+  it("comment + job mixed program → correct parse and compile", () => {
+    const source = '# header\njob etl {\n  find /world/x # query\n  | save /world/y\n}\n# footer';
+    const tokens = lex.tokenize(source);
+    // Should have JOB, IDENTIFIER, LBRACE, FIND, PATH, NEWLINE, PIPE, SAVE, PATH, NEWLINE, RBRACE, etc.
+    const meaningful = types(tokens).filter((x) => x !== "NEWLINE" && x !== "EOF");
+    expect(meaningful).toContain("JOB");
+    expect(meaningful).toContain("FIND");
+    expect(meaningful).toContain("SAVE");
+  });
+
+  // ── Bad ──
+
+  it("# inside string is NOT a comment", () => {
+    const tokens = lex.tokenize('"hello # world"');
+    expect(tokens[0]).toMatchObject({ type: "STRING", value: "hello # world" });
+  });
+
+  // ── Edge ──
+
+  it("only comments → empty program (just EOF)", () => {
+    const tokens = lex.tokenize("# only comments\n# nothing else");
+    const t = types(tokens).filter((x) => x !== "NEWLINE");
+    expect(t).toEqual(["EOF"]);
+  });
+
+  it("comment right after { or }", () => {
+    const tokens = lex.tokenize("{# comment\n}# comment");
+    const t = types(tokens).filter((x) => x !== "NEWLINE" && x !== "EOF");
+    expect(t).toEqual(["LBRACE", "RBRACE"]);
+  });
+
+  it("comment after pipe | → next line continues pipeline", () => {
+    const tokens = lex.tokenize("find /world/x | # filter next\nwhere active == true");
+    const t = types(tokens).filter((x) => x !== "NEWLINE" && x !== "EOF");
+    expect(t).toEqual(["FIND", "PATH", "PIPE", "WHERE", "IDENTIFIER", "EQ", "IDENTIFIER"]);
+  });
+
+  it("empty comment # → skipped", () => {
+    const tokens = lex.tokenize("#\njob");
+    const t = types(tokens).filter((x) => x !== "NEWLINE" && x !== "EOF");
+    expect(t).toEqual(["JOB"]);
+  });
+
+  // ── Security ──
+
+  it("code in comment # process.exit() → completely ignored", () => {
+    const tokens = lex.tokenize("# process.exit()\njob");
+    const t = types(tokens).filter((x) => x !== "NEWLINE" && x !== "EOF");
+    expect(t).toEqual(["JOB"]);
+  });
+
+  // ── Data Loss ──
+
+  it("comments do not break any existing v0 tests (regression)", () => {
+    // A v0 program without comments still works
+    const tokens = lex.tokenize('job etl { find /world/x | where active == true | save /world/y }');
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toContain("JOB");
+    expect(t).toContain("FIND");
+    expect(t).toContain("WHERE");
+    expect(t).toContain("SAVE");
+  });
+});
+
+describe("Phase 7: let/$ tokens", () => {
+  it("tokenize `let` → LET token", () => {
+    const tokens = lex.tokenize("let");
+    expect(tokens[0]).toMatchObject({ type: "LET", value: "let" });
+  });
+
+  it("tokenize `$varname` → DOLLAR + IDENTIFIER", () => {
+    const tokens = lex.tokenize("$threshold");
+    expect(tokens[0]).toMatchObject({ type: "DOLLAR", value: "$" });
+    expect(tokens[1]).toMatchObject({ type: "IDENTIFIER", value: "threshold" });
+  });
+
+  it("let x = 42 → LET IDENTIFIER ASSIGN NUMBER", () => {
+    const tokens = lex.tokenize("let x = 42");
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["LET", "IDENTIFIER", "ASSIGN", "NUMBER"]);
+  });
+});
+
+describe("Phase v3-0: action token", () => {
+  it("tokenize `action` → ACTION token", () => {
+    const tokens = lex.tokenize("action");
+    expect(tokens[0]).toMatchObject({ type: "ACTION", value: "action" });
+  });
+
+  it("tokenize `action /tools/classify` → ACTION PATH", () => {
+    const tokens = lex.tokenize("action /tools/classify");
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["ACTION", "PATH"]);
+  });
+
+  it("tokenize `action /tools/email { template: \"welcome\" }` → ACTION PATH LBRACE ... RBRACE", () => {
+    const tokens = lex.tokenize('action /tools/email { template: "welcome" }');
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["ACTION", "PATH", "LBRACE", "IDENTIFIER", "COLON", "STRING", "RBRACE"]);
+  });
+
+  it("tokenize `:` → COLON token", () => {
+    const tokens = lex.tokenize(":");
+    expect(tokens[0]).toMatchObject({ type: "COLON", value: ":" });
+  });
+
+  it("keyword prefix `actionable` → IDENTIFIER not ACTION", () => {
+    const tokens = lex.tokenize("actionable");
+    expect(tokens[0]).toMatchObject({ type: "IDENTIFIER", value: "actionable" });
+  });
+
+  it("action in pipeline: `find /data | action /tools/x | save /out`", () => {
+    const tokens = lex.tokenize("find /data | action /tools/x | save /out");
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["FIND", "PATH", "PIPE", "ACTION", "PATH", "PIPE", "SAVE", "PATH"]);
+  });
+});
+
+describe("Phase v3-1: expression operator tokens", () => {
+  it("tokenize `+` → PLUS token", () => {
+    const tokens = lex.tokenize("+");
+    expect(tokens[0]).toMatchObject({ type: "PLUS", value: "+" });
+  });
+
+  it("tokenize `-` → MINUS token", () => {
+    const tokens = lex.tokenize("-");
+    expect(tokens[0]).toMatchObject({ type: "MINUS", value: "-" });
+  });
+
+  it("tokenize `*` → STAR token", () => {
+    const tokens = lex.tokenize("*");
+    expect(tokens[0]).toMatchObject({ type: "STAR", value: "*" });
+  });
+
+  it("tokenize `/` alone (EOF after) → SLASH token", () => {
+    const tokens = lex.tokenize("/");
+    expect(tokens[0]).toMatchObject({ type: "SLASH", value: "/" });
+  });
+
+  it("tokenize `/ 2` → SLASH NUMBER", () => {
+    const tokens = lex.tokenize("/ 2");
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["SLASH", "NUMBER"]);
+  });
+
+  it("tokenize `/world/data` → PATH (still works)", () => {
+    const tokens = lex.tokenize("/world/data");
+    expect(tokens[0]).toMatchObject({ type: "PATH", value: "/world/data" });
+  });
+
+  it("operators in context: `map { x: a + b }` tokenizes correctly", () => {
+    const tokens = lex.tokenize("map { x: a + b }");
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["MAP", "LBRACE", "IDENTIFIER", "COLON", "IDENTIFIER", "PLUS", "IDENTIFIER", "RBRACE"]);
+  });
+
+  it("operators in context: `map { x: a * 0.8 + b }` tokenizes correctly", () => {
+    const tokens = lex.tokenize("map { x: a * 0.8 + b }");
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["MAP", "LBRACE", "IDENTIFIER", "COLON", "IDENTIFIER", "STAR", "NUMBER", "PLUS", "IDENTIFIER", "RBRACE"]);
+  });
+
+  it("operators in context: `map { x: a / b - c }` with SLASH not path", () => {
+    const tokens = lex.tokenize("map { x: a / b - c }");
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["MAP", "LBRACE", "IDENTIFIER", "COLON", "IDENTIFIER", "SLASH", "IDENTIFIER", "MINUS", "IDENTIFIER", "RBRACE"]);
+  });
+
+  it("group-by still works with - in compound keyword", () => {
+    const tokens = lex.tokenize("group-by dept");
+    expect(tokens[0]).toMatchObject({ type: "GROUP_BY", value: "group-by" });
+  });
+
+  it("paths with dot prefix still work: `/../etc`", () => {
+    const tokens = lex.tokenize("/../etc");
+    expect(tokens[0]).toMatchObject({ type: "PATH", value: "/../etc" });
+  });
+});
+
+describe("Phase v3-2: route tokens", () => {
+  it("tokenize `route` → ROUTE token", () => {
+    const tokens = lex.tokenize("route");
+    expect(tokens[0]).toMatchObject({ type: "ROUTE", value: "route" });
+  });
+
+  it("tokenize `->` → ARROW token", () => {
+    const tokens = lex.tokenize("->");
+    expect(tokens[0]).toMatchObject({ type: "ARROW", value: "->" });
+  });
+
+  it("`-` alone → MINUS, not ARROW", () => {
+    const tokens = lex.tokenize("- 1");
+    expect(tokens[0]).toMatchObject({ type: "MINUS", value: "-" });
+  });
+
+  it("route branch context tokenizes correctly", () => {
+    const tokens = lex.tokenize('route category { "urgent" -> job handle }');
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["ROUTE", "IDENTIFIER", "LBRACE", "STRING", "ARROW", "JOB", "IDENTIFIER", "RBRACE"]);
+  });
+});
+
+describe("Phase 8: count/group-by tokens", () => {
+  it("tokenize `count` → COUNT token", () => {
+    const tokens = lex.tokenize("count");
+    expect(tokens[0]).toMatchObject({ type: "COUNT", value: "count" });
+  });
+
+  it("tokenize `group` → IDENTIFIER (not keyword alone)", () => {
+    const tokens = lex.tokenize("group");
+    expect(tokens[0]).toMatchObject({ type: "IDENTIFIER", value: "group" });
+  });
+
+  it("tokenize pipeline with count: find | count", () => {
+    const tokens = lex.tokenize("find /world/x | count");
+    const t = types(tokens).filter((x) => x !== "EOF");
+    expect(t).toEqual(["FIND", "PATH", "PIPE", "COUNT"]);
+  });
+});
+
+describe("Phase v3-2: lookup tokens", () => {
+  it("tokenizes `lookup` as LOOKUP keyword", () => {
+    const tokens = lex.tokenize("lookup");
+    expect(tokens[0]).toMatchObject({ type: "LOOKUP", value: "lookup" });
+  });
+
+  it("tokenizes `lookup /data/customers on customer_id`", () => {
+    const tokens = lex.tokenize("lookup /data/customers on customer_id");
+    const t = types(tokens).filter(x => x !== "EOF");
+    expect(t).toEqual(["LOOKUP", "PATH", "ON", "IDENTIFIER"]);
+  });
+});
+
+describe("Template Parameters: Lexer", () => {
+  // STRING templates
+  it("string with ${name} preserves template in token value", () => {
+    const tokens = lex.tokenize('"hello ${name}"');
+    expect(tokens[0]).toMatchObject({ type: "STRING", value: "hello ${name}" });
+  });
+
+  it("string with escaped \\${literal} preserves \\$ in token value", () => {
+    const tokens = lex.tokenize('"hello \\${literal}"');
+    expect(tokens[0]).toMatchObject({ type: "STRING", value: "hello \\${literal}" });
+  });
+
+  it("string with multiple templates preserves all", () => {
+    const tokens = lex.tokenize('"${a} and ${b}"');
+    expect(tokens[0]).toMatchObject({ type: "STRING", value: "${a} and ${b}" });
+  });
+
+  it("string with nested field template ${data.id}", () => {
+    const tokens = lex.tokenize('"id=${data.id}"');
+    expect(tokens[0]).toMatchObject({ type: "STRING", value: "id=${data.id}" });
+  });
+
+  // PATH templates
+  it("path with mid-path template: /store/${category}/.actions/save", () => {
+    const tokens = lex.tokenize("/store/${category}/.actions/save");
+    expect(tokens[0]).toMatchObject({ type: "PATH", value: "/store/${category}/.actions/save" });
+  });
+
+  it("path with nested field template: /msgs/${data.messageId}", () => {
+    const tokens = lex.tokenize("/msgs/${data.messageId}");
+    expect(tokens[0]).toMatchObject({ type: "PATH", value: "/msgs/${data.messageId}" });
+  });
+
+  it("path with multiple templates: /conv/${convId}/msgs/${msgId}", () => {
+    const tokens = lex.tokenize("/conv/${convId}/msgs/${msgId}");
+    expect(tokens[0]).toMatchObject({ type: "PATH", value: "/conv/${convId}/msgs/${msgId}" });
+  });
+
+  it("path with template followed by params: action /send/${id}/.actions/do { text: \"hi\" }", () => {
+    const tokens = lex.tokenize('action /send/${id}/.actions/do { text: "hi" }');
+    const t = types(tokens).filter(x => x !== "EOF");
+    expect(t).toEqual(["ACTION", "PATH", "LBRACE", "IDENTIFIER", "COLON", "STRING", "RBRACE"]);
+    expect(tokens[1].value).toBe("/send/${id}/.actions/do");
+  });
+});
+
+describe("Phase v3-3: param tokens", () => {
+  it("tokenizes `param` as PARAM keyword", () => {
+    const tokens = lex.tokenize("param");
+    expect(tokens[0]).toMatchObject({ type: "PARAM", value: "param" });
+  });
+
+  it("tokenizes `param source = /data/users`", () => {
+    const tokens = lex.tokenize("param source = /data/users");
+    const t = types(tokens).filter(x => x !== "EOF");
+    expect(t).toEqual(["PARAM", "IDENTIFIER", "ASSIGN", "PATH"]);
+  });
+
+  it("tokenizes `param threshold = 80`", () => {
+    const tokens = lex.tokenize("param threshold = 80");
+    const t = types(tokens).filter(x => x !== "EOF");
+    expect(t).toEqual(["PARAM", "IDENTIFIER", "ASSIGN", "NUMBER"]);
+  });
+});
+
+describe("Phase v3-3: on trigger tokens", () => {
+  it("tokenizes `on` as ON keyword", () => {
+    const tokens = lex.tokenize("on");
+    expect(tokens[0]).toMatchObject({ type: "ON", value: "on" });
+  });
+
+  it("tokenizes `job handle on /data/inbox:created { }` trigger context", () => {
+    const tokens = lex.tokenize("job handle on /data/inbox:created { }");
+    const t = types(tokens).filter(x => x !== "EOF");
+    expect(t).toEqual(["JOB", "IDENTIFIER", "ON", "PATH", "COLON", "IDENTIFIER", "LBRACE", "RBRACE"]);
+  });
+
+  it("trigger with deep path: `on /a/b/c/d:updated`", () => {
+    const tokens = lex.tokenize("on /a/b/c/d:updated");
+    const t = types(tokens).filter(x => x !== "EOF");
+    expect(t).toEqual(["ON", "PATH", "COLON", "IDENTIFIER"]);
+  });
+});