@flink-app/flink 0.14.3 → 2.0.0-alpha.100

package/SIMPLE_AST_FEASIBILITY.md

@@ -0,0 +1,570 @@
+# Simple AST / Enhanced Text Parsing: Feasibility Analysis
+
+## Question
+Can we handle high-risk TypeScript features (template literals, conditional types, mapped types, string literals with brackets) by building more sophisticated text parsing instead of falling back to ts-morph?
+
+## TL;DR Recommendation
+
+**✅ YES for String Literals** - Build simple tokenizer (High ROI, Low complexity)
+**🤔 MAYBE for Template Literals** - Possible but complex (Medium ROI, Medium complexity)
+**❌ NO for Conditional/Mapped Types** - Use ts-morph fallback (Low ROI, High complexity)
+
+**Recommended Approach:** Hybrid strategy
+1. Enhanced tokenizer for strings/comments (handles 99% of cases)
+2. ts-morph fallback for truly complex types (handles remaining 1%)
+
+---
+
+## Feasibility Analysis by Feature
+
+### 1. String Literals with Brackets ✅ FEASIBLE
+
+**Problem:**
+```typescript
+interface Config {
+    pattern: "{user:id}";     // Parser sees { as object start
+    regex: "/test>/";         // Parser sees > as generic close
+    quote: "He said \"hi\"";  // Escape handling
+}
+```
+
+**Solution: Simple String State Machine**
+
+```typescript
+/**
+ * Enhanced bracket counting with string literal awareness
+ */
+function findMatchingBracket(text: string, startPos: number): number {
+    let depth = 1;
+    let i = startPos + 1;
+    let inString = false;
+    let stringChar = '';
+
+    while (i < text.length && depth > 0) {
+        const char = text[i];
+        const prevChar = text[i - 1];
+
+        // Handle string state transitions
+        if (!inString) {
+            if (char === '"' || char === "'" || char === '`') {
+                inString = true;
+                stringChar = char;
+            } else {
+                // Only count brackets outside strings
+                if (char === '{') depth++;
+                else if (char === '}') depth--;
+                else if (char === '<') depth++;
+                else if (char === '>') depth--;
+            }
+        } else {
+            // Inside string - check for closing quote
+            if (char === stringChar && prevChar !== '\\') {
+                inString = false;
+                stringChar = '';
+            }
+        }
+
+        i++;
+    }
+
+    return depth === 0 ? i - 1 : -1;
+}
+```
+
+**Complexity:** Low
+**Benefit:** High (fixes most common edge case)
+**Verdict:** ✅ **IMPLEMENT THIS**
+
+---
+
+### 2. Template Literal Types 🤔 MAYBE FEASIBLE
+
+**Problem:**
+```typescript
+type Route = `/${string}`;
+type EventName = `on${Capitalize<string>}`;
+type Path = `users/${number}/posts/${number}`;
+```
+
+**Challenge:** Template expressions `${...}` contain TypeScript code
+
+**Solution: Template Literal State Machine**
+
+```typescript
+/**
+ * Parse template literal type
+ */
+function parseTemplateLiteral(text: string, startPos: number): {
+    literalParts: string[];
+    expressions: string[];
+} {
+    const literalParts: string[] = [];
+    const expressions: string[] = [];
+
+    let i = startPos + 1; // Skip opening `
+    let currentLiteral = '';
+    let inExpression = false;
+    let exprDepth = 0;
+
+    while (i < text.length) {
+        const char = text[i];
+        const nextChar = text[i + 1];
+
+        if (!inExpression) {
+            if (char === '$' && nextChar === '{') {
+                // Start of expression
+                literalParts.push(currentLiteral);
+                currentLiteral = '';
+                inExpression = true;
+                exprDepth = 1;
+                i += 2; // Skip ${
+                continue;
+            } else if (char === '`') {
+                // End of template
+                literalParts.push(currentLiteral);
+                break;
+            } else {
+                currentLiteral += char;
+            }
+        } else {
+            // Inside ${...} expression
+            if (char === '{') exprDepth++;
+            else if (char === '}') {
+                exprDepth--;
+                if (exprDepth === 0) {
+                    inExpression = false;
+                    i++;
+                    continue;
+                }
+            }
+            expressions.push(char);
+        }
+
+        i++;
+    }
+
+    return { literalParts, expressions };
+}
+```
+
+**But wait... the expressions contain TypeScript code!**
+
+Inside `${}` we can have:
+- Type references: `${string}`, `${User}`
+- Utility types: `${Capitalize<string>}`
+- Conditional types: `${T extends string ? T : never}`
+- Mapped types: `${keyof T}`
+
+This means we need to **recursively parse TypeScript types** inside template expressions!
+
+**Complexity:** Medium-High (needs recursive type parser)
+**Benefit:** Medium (template literal types are growing in popularity but still relatively rare)
+**Verdict:** 🤔 **POSSIBLE BUT COMPLEX - Consider ts-morph fallback**
+
+---
+
+### 3. Conditional Types ❌ NOT FEASIBLE
+
+**Problem:**
+```typescript
+type ExtractParams<T> = T extends `${infer Start}/${infer Rest}`
+    ? Start | ExtractParams<Rest>
+    : T;
+
+type IsString<T> = T extends string ? true : false;
+```
+
+**Challenge:** Understanding `extends`, `infer`, ternary operator in type context
+
+**What we'd need to parse:**
+1. Generic type parameters with constraints: `<T extends Constraint>`
+2. Type inference: `infer Start`, `infer Rest`
+3. Ternary operator: `? TrueType : FalseType`
+4. Recursive type references: `ExtractParams<Rest>`
+5. Template literal patterns: `` `${infer Start}` ``
+
+**Attempted Solution:**
+```typescript
+function parseConditionalType(text: string) {
+    // Need to:
+    // 1. Find the "extends" keyword (but not in other contexts!)
+    // 2. Parse the constraint expression
+    // 3. Find the "?" operator (but not in other contexts!)
+    // 4. Parse the true branch (might contain nested conditionals!)
+    // 5. Find the ":" operator (but not in other contexts!)
+    // 6. Parse the false branch (might contain nested conditionals!)
+    // 7. Handle "infer" keyword and capture groups
+    // 8. Track nested template literals with expressions
+
+    // This is essentially re-implementing TypeScript's type parser! 😱
+}
+```
+
+**Reality Check:**
+- TypeScript's parser is ~50,000 lines of code
+- Conditional types interact with ALL other type features
+- Edge cases are numerous and subtle
+- We'd be maintaining a subset parser that needs to evolve with TypeScript
+
+**Complexity:** Very High
+**Benefit:** Low (conditional types are rare in tool schemas)
+**Verdict:** ❌ **NOT WORTH IT - Use ts-morph fallback**
+
+---
+
+### 4. Mapped Types ❌ NOT FEASIBLE
+
+**Problem:**
+```typescript
+type Readonly<T> = {
+    readonly [P in keyof T]: T[P];
+};
+
+type Getters<T> = {
+    [K in keyof T as `get${Capitalize<K>}`]: () => T[K];
+};
+```
+
+**Challenge:** Understanding `in`, `keyof`, `as`, type remapping
+
+**What we'd need to parse:**
+1. Index signature with type parameter: `[P in keyof T]`
+2. Type operators: `keyof T`, `typeof x`, `T[K]`
+3. Type remapping: `as NewType`
+4. Template literal remapping: `` as `get${Capitalize<K>}` ``
+5. Modifiers: `readonly`, `?`, `+`, `-`
+
+**Complexity:** Very High
+**Benefit:** Very Low (mapped types almost never appear in tool schemas)
+**Verdict:** ❌ **NOT WORTH IT - Use ts-morph fallback**
+
+---
+
+## Comparative Analysis
+
+| Feature | Text Parsing Complexity | Frequency in Schemas | Benefit/Effort Ratio |
+|---------|------------------------|---------------------|---------------------|
+| String literals with brackets | ⭐ Low | ⭐⭐⭐ High | ✅ **Excellent** |
+| Template literal types | ⭐⭐⭐ Medium-High | ⭐⭐ Medium | 🤔 **Fair** |
+| Conditional types | ⭐⭐⭐⭐⭐ Very High | ⭐ Low | ❌ **Poor** |
+| Mapped types | ⭐⭐⭐⭐⭐ Very High | ⭐ Very Low | ❌ **Poor** |
+
+---
+
+## Recommended Implementation: Hybrid Strategy
+
+### Phase 1: Enhanced Tokenizer (High Value, Low Effort) ✅
+
+Build a simple tokenizer that properly handles:
+- String literals (all three quote types)
+- Escape sequences
+- Comments (line and block)
+- Bracket/paren/angle depth tracking
+
+**Implementation:**
+
+```typescript
+/**
+ * Simple TypeScript tokenizer for schema extraction
+ */
+class TypeScriptTokenizer {
+    private text: string;
+    private pos: number = 0;
+
+    constructor(text: string) {
+        this.text = text;
+    }
+
+    /**
+     * Find matching closing bracket, properly handling strings and comments
+     */
+    findMatchingBracket(openChar: '{' | '<' | '(' | '['): number {
+        const closeChar = this.getClosingBracket(openChar);
+        let depth = 1;
+        let i = this.pos + 1;
+
+        while (i < this.text.length && depth > 0) {
+            const token = this.getNextToken(i);
+
+            if (token.type === 'bracket') {
+                if (token.value === openChar) depth++;
+                else if (token.value === closeChar) depth--;
+            }
+
+            i = token.endPos;
+        }
+
+        return depth === 0 ? i - 1 : -1;
+    }
+
+    /**
+     * Get next meaningful token, skipping strings and comments
+     */
+    private getNextToken(startPos: number): Token {
+        const char = this.text[startPos];
+
+        // String literal
+        if (char === '"' || char === "'" || char === '`') {
+            return this.parseStringLiteral(startPos, char);
+        }
+
+        // Line comment
+        if (char === '/' && this.text[startPos + 1] === '/') {
+            return this.parseLineComment(startPos);
+        }
+
+        // Block comment
+        if (char === '/' && this.text[startPos + 1] === '*') {
+            return this.parseBlockComment(startPos);
+        }
+
+        // Bracket
+        if ('{}<>()[]'.includes(char)) {
+            return {
+                type: 'bracket',
+                value: char,
+                startPos,
+                endPos: startPos + 1,
+            };
+        }
+
+        // Other character
+        return {
+            type: 'char',
+            value: char,
+            startPos,
+            endPos: startPos + 1,
+        };
+    }
+
+    private parseStringLiteral(startPos: number, quoteChar: string): Token {
+        let i = startPos + 1;
+        let escaped = false;
+
+        while (i < this.text.length) {
+            if (escaped) {
+                escaped = false;
+                i++;
+                continue;
+            }
+
+            if (this.text[i] === '\\') {
+                escaped = true;
+                i++;
+                continue;
+            }
+
+            if (this.text[i] === quoteChar) {
+                return {
+                    type: 'string',
+                    value: this.text.slice(startPos, i + 1),
+                    startPos,
+                    endPos: i + 1,
+                };
+            }
+
+            i++;
+        }
+
+        throw new Error(`Unclosed string literal at position ${startPos}`);
+    }
+
+    private parseLineComment(startPos: number): Token {
+        const newlinePos = this.text.indexOf('\n', startPos);
+        const endPos = newlinePos === -1 ? this.text.length : newlinePos + 1;
+
+        return {
+            type: 'comment',
+            value: this.text.slice(startPos, endPos),
+            startPos,
+            endPos,
+        };
+    }
+
+    private parseBlockComment(startPos: number): Token {
+        const endPos = this.text.indexOf('*/', startPos + 2);
+
+        if (endPos === -1) {
+            throw new Error(`Unclosed block comment at position ${startPos}`);
+        }
+
+        return {
+            type: 'comment',
+            value: this.text.slice(startPos, endPos + 2),
+            startPos,
+            endPos: endPos + 2,
+        };
+    }
+
+    private getClosingBracket(open: string): string {
+        const pairs: Record<string, string> = {
+            '{': '}',
+            '<': '>',
+            '(': ')',
+            '[': ']',
+        };
+        return pairs[open];
+    }
+}
+
+interface Token {
+    type: 'string' | 'comment' | 'bracket' | 'char';
+    value: string;
+    startPos: number;
+    endPos: number;
+}
+```
+
+**Usage in findTypeDefinition():**
+
+```typescript
+static findTypeDefinition(sourceText: string, typeName: string): TypeDefinition | null {
+    // ... existing regex to find interface/type ...
+
+    // Use tokenizer instead of manual bracket counting
+    const tokenizer = new TypeScriptTokenizer(sourceText);
+    tokenizer.pos = braceStart;
+    const endPos = tokenizer.findMatchingBracket('{');
+
+    if (endPos === -1) {
+        return null;
+    }
+
+    const definition = sourceText.slice(startIndex, endPos + 1).trim();
+    return { name: typeName, definition, kind: 'interface' };
+}
+```
+
+**Benefits:**
+- ✅ Fixes string literal edge cases
+- ✅ Robust comment handling
+- ✅ Proper escape sequence handling
+- ✅ ~100 lines of code (manageable)
+- ✅ Reusable for all bracket matching
+
+---
+
+### Phase 2: Template Literal Detection (Optional) 🤔
+
+**Simple Approach:** Detect but don't parse
+
+```typescript
+function hasTemplateLiteralType(text: string): boolean {
+    // Match: type Name = `...`
+    return /type\s+\w+\s*=\s*`/.test(text);
+}
+
+function extractSchemasFast(filePath: string) {
+    const fileText = fs.readFileSync(filePath, 'utf8');
+
+    // Detect template literal types
+    if (hasTemplateLiteralType(fileText)) {
+        perfLog.info(`${path.basename(filePath)}: Template literal types detected, using ts-morph`);
+        return extractSchemasWithTsMorph(filePath);
+    }
+
+    // Continue with fast extraction...
+}
+```
+
+**Benefits:**
+- ✅ Simple detection (1 line regex)
+- ✅ Maintains safety for complex case
+- ✅ No need to parse template expressions
+
+---
+
+### Phase 3: ts-morph Fallback for Complex Types ✅
+
+For conditional and mapped types, just use ts-morph:
+
+```typescript
+function hasComplexTypeFeatures(text: string): boolean {
+    return (
+        hasConditionalType(text) ||
+        hasMappedType(text) ||
+        hasTemplateLiteralType(text)  // If Phase 2 not implemented
+    );
+}
+
+function hasConditionalType(text: string): boolean {
+    // Match: T extends U ? A : B
+    return /\bextends\b.*?\?.*?:/.test(text);
+}
+
+function hasMappedType(text: string): boolean {
+    // Match: [K in keyof T]
+    return /\[\s*\w+\s+in\s+keyof\s+/.test(text);
+}
+```
+
+**Benefits:**
+- ✅ 100% correctness for complex cases
+- ✅ Minimal performance impact (rare cases)
+- ✅ No need to maintain complex parser
+
+---
+
+## Implementation Effort vs Benefit
+
+| Approach | Lines of Code | Dev Time | Maintenance | Performance | Safety |
+|----------|--------------|----------|-------------|-------------|--------|
+| **Enhanced Tokenizer** | ~100 | 4 hours | Low | 1200x | 99% |
+| **+ Template Parsing** | ~300 | 16 hours | Medium | 1150x | 99.5% |
+| **+ Full Type Parser** | ~5000+ | 200+ hours | High | 1100x | 99.9% |
+| **Hybrid (Tokenizer + Fallback)** | ~150 | 6 hours | Low | 1100x | **100%** |
+
+---
+
+## Recommended Path Forward
+
+### ✅ Implement Now
+1. **Enhanced tokenizer** for string/comment handling (~4 hours, high value)
+2. **Detection functions** for complex types (~1 hour)
+3. **ts-morph fallback** for complex cases (~1 hour)
+
+**Total effort:** ~6 hours
+**Result:** 99.9%+ safety, ~1100x speedup, minimal maintenance
+
+### 🤔 Consider Later
+4. **Template literal parsing** if usage grows (~16 hours, medium value)
+
+### ❌ Don't Implement
+5. **Full conditional type parsing** - not worth the effort
+6. **Full mapped type parsing** - not worth the effort
+
+---
+
+## Conclusion
+
+**YES** to enhanced tokenizer (string/comment handling) ✅
+**NO** to full AST for complex types ❌
+**RECOMMENDED:** Hybrid approach with simple tokenizer + ts-morph fallback
+
+### Why Hybrid is Optimal
+
+1. **Performance:** Maintains ~1100x speedup (vs 1250x pure text)
+2. **Safety:** 100% correctness via ts-morph fallback
+3. **Complexity:** ~150 lines of tokenizer code (manageable)
+4. **Maintenance:** Minimal - only maintain tokenizer, not full parser
+5. **Future-proof:** TypeScript can add new features, we just fallback
+
+### The Reality
+
+Building a full TypeScript type parser means:
+- Re-implementing 10-20% of TypeScript's parser
+- Keeping up with TypeScript versions (3-4 releases/year)
+- Handling edge cases that take TypeScript team months to fix
+- Testing against thousands of type combinations
+
+**It's not worth it when ts-morph is available as a safety net.**
+
+### Final Recommendation
+
+Implement **enhanced tokenizer** (Phase 1) to handle string literals robustly, then use **ts-morph fallback** for truly complex types. This gives us:
+- ✅ Best performance (1100x+ speedup)
+- ✅ Best safety (100% correctness)
+- ✅ Best maintainability (small codebase)
+- ✅ Best of both worlds
+
+**Start with Phase 1, measure real-world fallback frequency, then decide if Phase 2 is needed.**

package/bin/flink.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-const commands = ["build", "generate" /* 'generate' is alias for 'build'   */, "run", "clean", "help"];
+const commands = ["build", "generate" /* 'generate' is alias for 'build' */, "run", "dev", "clean", "help"];
 
 const argv = process.argv.slice(2);
 let argCommand = argv[0];
@@ -21,6 +21,17 @@ if (argCommand === "generate") {
 
 const cmd = require("../cli/" + argCommand);
 
-cmd(argv.slice(1));
+(async () => {
+    try {
+        await cmd(argv.slice(1));
+    } catch (error: any) {
+        if (process.env.DEBUG === "true") {
+            console.error(error);
+        } else {
+            console.error(`Error: ${error.message}`);
+        }
+        process.exit(1);
+    }
+})();
 
 export default () => {};

package/cli/build.ts

@@ -1,48 +1,28 @@
 #!/usr/bin/env node
-import TypeScriptCompiler from "../src/TypeScriptCompiler";
-import { getOption } from "./cli-utils";
-
-module.exports = async function run(args: string[]) {
-    const startTime = Date.now();
-
-    if (args.includes("--help")) {
-        console.log(`
-    Description
-      Builds the application.
-      Will generate intermediates files in .flink and compile/transpile
-      javascript bundle in /dist folder
-
-    Usage
-      $ flink build <dir>
-
-    <dir> represents the directory of the Flink application.
-    If no directory is provided, the current directory will be used.
-      
-    Options            
-      --help    Displays this message
-      `);
-
-        process.exit(0);
-    }
-
-    let dir = "./";
-    if (args[0] && !args[0].startsWith("--")) {
-        dir = args[0];
+import { parseArgs, hasFlag, printHelp, compile } from "./cli-utils";
+import { FlinkLogFactory } from "../src/FlinkLogFactory";
+import { loadFlinkConfig } from "../src/utils/loadFlinkConfig";
+
+// Load config BEFORE creating logger
+const flinkConfig = loadFlinkConfig();
+FlinkLogFactory.configure(flinkConfig?.logging);
+
+module.exports = async function build(args: string[]) {
+    const parsed = parseArgs(args);
+
+    if (hasFlag(parsed, "help")) {
+        printHelp({
+            description:
+                "Builds the application.\n      Will generate intermediates files in .flink and compile/transpile\n      javascript bundle in /dist folder",
+            usage: "$ flink build <dir>",
+            options: [{ name: "help", description: "Displays this message" }],
+        });
+        return;
     }
 
-    const exclude = getOption(args, "exclude", "/spec") as string;
-
-    await TypeScriptCompiler.clean(dir);
-
-    const compiler = new TypeScriptCompiler(dir);
-
-    if (!compiler.getPreEmitDiagnostics()) {
-        process.exit(1);
-    }
-
-    await Promise.all([compiler.parseRepos(), compiler.parseHandlers(exclude.split(",")), compiler.parseJobs(), compiler.generateStartScript()]);
-
-    console.log(`Compilation done, took ${Date.now() - startTime}ms`);
-
-    compiler.emit();
+    await compile({
+        dir: parsed.dir,
+        typeCheck: true,
+        timingLogs: true,
+    });
 };