@flink-app/flink 0.14.3 → 2.0.0-alpha.100

package/spec/TypeScriptTokenizer.spec.ts

@@ -0,0 +1,366 @@
+import { TypeScriptTokenizer } from "../src/schema-extraction/TypeScriptTokenizer";
+
+describe("TypeScriptTokenizer", () => {
+    describe("findMatchingBracket", () => {
+        describe("Basic bracket matching", () => {
+            it("should find matching closing brace for simple object", () => {
+                const code = "{ foo: string }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(14);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should find matching closing bracket for array", () => {
+                const code = "[1, 2, 3]";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "[");
+                expect(endPos).toBe(8);
+                expect(code[endPos]).toBe("]");
+            });
+
+            it("should find matching closing paren", () => {
+                const code = "(foo, bar)";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "(");
+                expect(endPos).toBe(9);
+                expect(code[endPos]).toBe(")");
+            });
+
+            it("should find matching closing angle bracket", () => {
+                const code = "<T extends string>";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "<");
+                expect(endPos).toBe(17);
+                expect(code[endPos]).toBe(">");
+            });
+        });
+
+        describe("Nested brackets", () => {
+            it("should handle nested objects", () => {
+                const code = "{ outer: { inner: string } }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(27);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle deeply nested objects", () => {
+                const code = "{ a: { b: { c: { d: string } } } }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(33);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle nested generics", () => {
+                const code = "<Promise<Array<string>>>";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "<");
+                expect(endPos).toBe(23);
+                expect(code[endPos]).toBe(">");
+            });
+
+            it("should handle mixed bracket types", () => {
+                const code = "{ arr: [1, 2], fn: () => {} }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(28);
+                expect(code[endPos]).toBe("}");
+            });
+        });
+
+        describe("String literals", () => {
+            it("should ignore brackets inside double-quoted strings", () => {
+                const code = '{ pattern: "{user:id}" }';
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(23);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should ignore brackets inside single-quoted strings", () => {
+                const code = "{ pattern: '{user:id}' }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(23);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should ignore brackets inside backtick strings", () => {
+                const code = "{ pattern: `{user:id}` }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(23);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should ignore angle brackets inside strings", () => {
+                const code = '{ regex: "/test>/" }';
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(19);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should ignore square brackets inside strings", () => {
+                const code = '{ pattern: "[0-9]+" }';
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(20);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle multiple string literals with brackets", () => {
+                const code = '{ a: "{foo}", b: "[bar]", c: "<baz>" }';
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(37);
+                expect(code[endPos]).toBe("}");
+            });
+        });
+
+        describe("Escape sequences", () => {
+            it("should handle escaped quotes", () => {
+                const code = '{ msg: "He said \\"hi\\"" }';
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(24);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle escaped backslashes", () => {
+                const code = '{ path: "C:\\\\Users\\\\test" }';
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(26);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle escaped brackets inside strings", () => {
+                const code = '{ pattern: "\\{escaped\\}" }';
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(25);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle complex escape sequences", () => {
+                const code = '{ regex: "\\n\\t\\r\\{\\}\\[\\]" }';
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(26);
+                expect(code[endPos]).toBe("}");
+            });
+        });
+
+        describe("Line comments", () => {
+            it("should ignore brackets in line comments", () => {
+                const code = "{ // This { is a comment\n  foo: string }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(39);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle multiple line comments", () => {
+                const code = `{
+  // Comment with {
+  foo: string, // Another { comment
+  bar: number
+}`;
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle line comment at end of file", () => {
+                const code = "{ foo: string } // { comment";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(14);
+                expect(code[endPos]).toBe("}");
+            });
+        });
+
+        describe("Block comments", () => {
+            it("should ignore brackets in block comments", () => {
+                const code = "{ /* This { is a comment */ foo: string }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(40);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle multi-line block comments", () => {
+                const code = `{
+  /*
+   * Comment with {
+   * More brackets [
+   */
+  foo: string
+}`;
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle multiple block comments", () => {
+                const code = "{ /* { */ foo: string, /* [ */ bar: number }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(43);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle nested block comment syntax", () => {
+                const code = "{ /* /* not nested in JS */ foo: string }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(40);
+                expect(code[endPos]).toBe("}");
+            });
+        });
+
+        describe("Complex real-world cases", () => {
+            it("should handle Google Docs tool schema case", () => {
+                const code = `interface Tool {
+  metadata?: {
+    title: string;
+    description: string;
+  };
+  execute: () => Promise<Result>;
+}`;
+                const tokenizer = new TypeScriptTokenizer(code);
+                const startPos = code.indexOf("{");
+                const endPos = tokenizer.findMatchingBracketAt(startPos, "{");
+                expect(code[endPos]).toBe("}");
+                expect(code.substring(startPos, endPos + 1)).toContain("metadata");
+                expect(code.substring(startPos, endPos + 1)).toContain("execute");
+            });
+
+            it("should handle handler with string patterns", () => {
+                const code = `{
+  route: "/api/{id}/items",
+  validation: {
+    pattern: "[0-9]+",
+    format: "<xml>"
+  }
+}`;
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle interface with comments and strings", () => {
+                const code = `{
+  // Route pattern: /api/{id}
+  path: "/users/{id}",
+  /* Validation rules {
+     complex: true
+  } */
+  validate: true
+}`;
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle generic type with nested brackets and strings", () => {
+                const code = `FlinkTool<Context, { pattern: "{id}" }, ToolResult<Output>>`;
+                const tokenizer = new TypeScriptTokenizer(code);
+                const startPos = code.indexOf("<");
+                const endPos = tokenizer.findMatchingBracketAt(startPos, "<");
+                expect(endPos).toBe(code.length - 1);
+                expect(code[endPos]).toBe(">");
+            });
+        });
+
+        describe("Edge cases", () => {
+            it("should return -1 for unmatched opening bracket", () => {
+                const code = "{ foo: string";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(-1);
+            });
+
+            it("should handle empty braces", () => {
+                const code = "{}";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(endPos).toBe(1);
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle whitespace-only content", () => {
+                const code = "{   \n  \t  }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                expect(code[endPos]).toBe("}");
+            });
+
+            it("should handle unclosed string literal gracefully", () => {
+                const code = '{ msg: "unclosed string }';
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                // Unclosed string extends to end, so no matching brace found
+                expect(endPos).toBe(-1);
+            });
+
+            it("should handle unclosed block comment gracefully", () => {
+                const code = "{ /* unclosed comment }";
+                const tokenizer = new TypeScriptTokenizer(code);
+                const endPos = tokenizer.findMatchingBracketAt(0, "{");
+                // Unclosed comment extends to end, so no matching brace found
+                expect(endPos).toBe(-1);
+            });
+        });
+
+        describe("getNextToken", () => {
+            it("should tokenize string literals", () => {
+                const tokenizer = new TypeScriptTokenizer('"hello"');
+                const token = tokenizer.getNextToken(0);
+                expect(token.type).toBe("string");
+                expect(token.value).toBe('"hello"');
+                expect(token.startPos).toBe(0);
+                expect(token.endPos).toBe(7);
+            });
+
+            it("should tokenize line comments", () => {
+                const tokenizer = new TypeScriptTokenizer("// comment\n");
+                const token = tokenizer.getNextToken(0);
+                expect(token.type).toBe("comment");
+                expect(token.value).toBe("// comment\n");
+                expect(token.startPos).toBe(0);
+                expect(token.endPos).toBe(11);
+            });
+
+            it("should tokenize block comments", () => {
+                const tokenizer = new TypeScriptTokenizer("/* comment */");
+                const token = tokenizer.getNextToken(0);
+                expect(token.type).toBe("comment");
+                expect(token.value).toBe("/* comment */");
+                expect(token.startPos).toBe(0);
+                expect(token.endPos).toBe(13);
+            });
+
+            it("should tokenize bracket characters", () => {
+                const tokenizer = new TypeScriptTokenizer("{");
+                const token = tokenizer.getNextToken(0);
+                expect(token.type).toBe("bracket");
+                expect(token.value).toBe("{");
+                expect(token.startPos).toBe(0);
+                expect(token.endPos).toBe(1);
+            });
+
+            it("should tokenize other characters", () => {
+                const tokenizer = new TypeScriptTokenizer("x");
+                const token = tokenizer.getNextToken(0);
+                expect(token.type).toBe("char");
+                expect(token.value).toBe("x");
+                expect(token.startPos).toBe(0);
+                expect(token.endPos).toBe(1);
+            });
+        });
+    });
+});