@comprehend/telemetry-node 0.1.3 → 0.2.0

package/src/sql-analyzer.ts

@@ -1,439 +0,0 @@
-type Token = { type: 'keyword' | 'identifier' | 'id-quote' | 'string' | 'comment' | 'punct' | 'operator' | 'whitespace' | 'unknown'; value: string };
-
-const KEYWORDS = new Set([
-    'SELECT', 'FROM', 'WHERE', 'INSERT', 'REPLACE', 'INTO', 'VALUES', 'DELETE', 'UPDATE',
-    'MERGE', 'SET', 'JOIN', 'LEFT', 'RIGHT', 'FULL', 'OUTER', 'INNER', 'ON', 'AS', 'AND', 'OR',
-    'NOT', 'IS', 'NULL', 'IN', 'WITH', 'RECURSIVE', 'UNION', 'ALL',
-    'GROUP', 'BY', 'HAVING', 'ORDER', 'LIMIT', 'OFFSET', 'LATERAL', 'USING'
-]);
-
-export interface SQLAnalysisResult {
-    tableOperations: Record<string, string[]>;
-    normalizedQuery: string;
-    presentableQuery: string;
-}
-
-/** Performs a rough tokenization of the SQL, extracts the tables involved and the operations on them, and
- * produces two versions of the query:
- * - A normalized version for hashing purposes that does not account for whitespace, comments, and collapses
- *   IN clauses and VALUES clauses that might cause a cardinality explosion.
- * - A presentable version that only does the IN clause and VALUES clause collapsing */
-export function analyzeSQL(sql: string): SQLAnalysisResult {
-    let semanticTokens = new Array<Token>();
-    let presentableTokens = new Array<Token>();
-    let seekingInParen = false;
-    let analyzingIn = false;
-    let skippingIn = false;
-    let seekingValuesParen = false;
-    let skippingValues = false;
-    let lookingForCommaOrEnd = false;
-    let valuesDepth = 0;
-    let skippedWhitespace: Token[] = [];
-    for (let token of tokenizeSQL(sql)) {
-        switch (token.type) {
-            case "whitespace":
-            case "comment":
-            case "id-quote":
-                // Skip
-                break;
-            case "keyword":
-                // Normalize to uppercase.
-                semanticTokens.push({ type: "keyword", value: token.value.toUpperCase() });
-                break;
-            case "identifier":
-                // Normalize to lowercase.
-                semanticTokens.push({ type: "identifier", value: token.value.toLowerCase() });
-                break;
-            default:
-                semanticTokens.push(token);
-                break;
-        }
-
-        if (seekingInParen) {
-            // We saw IN, and now look for an opening (. Skip whitespace/comments, bail if anything else.
-            presentableTokens.push(token);
-            switch (token.type) {
-                case "comment":
-                case "whitespace":
-                    break;
-                case "punct":
-                    seekingInParen = false;
-                    analyzingIn = token.value === "(";
-                    break;
-                default:
-                    seekingInParen = false;
-                    break;
-            }
-        }
-        else if (analyzingIn) {
-            // We saw the opening paren of an IN. Pass over whitespace and comments. If we see a
-            // keyword we know it's not something to collapse, it's a sub-query. Otherwise, we
-            // enter skipping mode.
-            switch (token.type) {
-                case "comment":
-                case "whitespace":
-                    presentableTokens.push(token);
-                    break;
-                case "keyword":
-                case "punct": // maybe immediate ), certainly not a value
-                    presentableTokens.push(token);
-                    analyzingIn = false;
-                    break;
-                default:
-                    analyzingIn = false;
-                    skippingIn = true;
-                    presentableTokens.push({ type: "unknown", value: "..." })
-            }
-        }
-        else if (skippingIn) {
-            // Omit tokens until a closing ).
-            if (token.type === "punct" && token.value === ")") {
-                presentableTokens.push(token);
-                skippingIn = false;
-            }
-        }
-        else if (seekingValuesParen) {
-            // We saw VALUES, and now look for an opening (. Skip whitespace/comments, bail if anything else.
-            switch (token.type) {
-                case "comment":
-                case "whitespace":
-                    presentableTokens.push(token);
-                    break;
-                case "punct":
-                    if (token.value === "(") {
-                        // Just add the opening paren, "..." and closing paren - preserve original spacing
-                        presentableTokens.push(token);
-                        presentableTokens.push({ type: "unknown", value: "..." });
-                        presentableTokens.push({ type: "punct", value: ")" });
-                        seekingValuesParen = false;
-                        skippingValues = true;
-                        valuesDepth = 1;
-                    } else {
-                        // Not what we expected, go back to normal processing
-                        presentableTokens.push(token);
-                        seekingValuesParen = false;
-                    }
-                    break;
-                default:
-                    // Not what we expected, go back to normal processing
-                    presentableTokens.push(token);
-                    seekingValuesParen = false;
-                    break;
-            }
-        }
-        else if (skippingValues) {
-            // Skip everything until we've consumed all VALUES tuples
-            if (token.type === "punct") {
-                if (token.value === "(") {
-                    valuesDepth++;
-                } else if (token.value === ")") {
-                    valuesDepth--;
-                    if (valuesDepth === 0) {
-                        // This closes a tuple, check for comma indicating more tuples
-                        lookingForCommaOrEnd = true;
-                        skippingValues = false;
-                    }
-                }
-            }
-        }
-        else if (lookingForCommaOrEnd) {
-            // After closing a VALUES tuple, look for comma (more tuples) or end of VALUES
-            switch (token.type) {
-                case "comment":
-                case "whitespace":
-                    // Collect whitespace/comments while looking for comma or end
-                    skippedWhitespace.push(token);
-                    break;
-                case "punct":
-                    if (token.value === ",") {
-                        // More tuples coming, clear skipped whitespace and continue skipping
-                        skippedWhitespace = [];
-                        lookingForCommaOrEnd = false;
-                        skippingValues = true;
-                    } else {
-                        // Not a comma, so VALUES clause is done
-                        // Add back the skipped whitespace, then the current token
-                        presentableTokens.push(...skippedWhitespace);
-                        presentableTokens.push(token);
-                        skippedWhitespace = [];
-                        lookingForCommaOrEnd = false;
-                    }
-                    break;
-                default:
-                    // VALUES clause is done, resume normal processing
-                    // Add back the skipped whitespace, then the current token
-                    presentableTokens.push(...skippedWhitespace);
-                    presentableTokens.push(token);
-                    skippedWhitespace = [];
-                    lookingForCommaOrEnd = false;
-                    break;
-            }
-        }
-        else {
-            presentableTokens.push(token);
-            seekingInParen = token.type === "keyword" && token.value.toUpperCase() === "IN";
-            seekingValuesParen = token.type === "keyword" && token.value.toUpperCase() === "VALUES";
-        }
-    }
-    return {
-        ...analyzeSQLTokens(semanticTokens),
-        presentableQuery: presentableTokens.map(t => t.value).join("")
-    };
-}
-
-function* tokenizeSQL(sql: string): Generator<Token> {
-    const regex = /(--[^\n]*|\/\*[\s\S]*?\*\/)|('[^']*')|("(?:[^"]*)")|(`[^`]*`)|(\[[^\]]+\])|(\b[a-zA-Z_][\w$]*\b)|([(),;])|(<=|>=|<>|!=|=|<|>)|(\s+)|(\S)/g;
-    let match: RegExpExecArray | null;
-
-    while ((match = regex.exec(sql))) {
-        const [
-            full,
-            comment,         // 1
-            singleQuoted,    // 2: string literal
-            doubleQuoted,    // 3: "identifier"
-            backtickQuoted,  // 4: `identifier`
-            bracketQuoted,   // 5: [identifier]
-            word,            // 6
-            punct,           // 7
-            operator,        // 8
-            ws,              // 9
-            unknown          // 10
-        ] = match;
-        if (comment) {
-            yield { type: 'comment', value: comment };
-        }
-        else if (singleQuoted) {
-            yield { type: 'string', value: singleQuoted };
-        }
-        else if (doubleQuoted) {
-            yield { type: 'id-quote', value: '"' };
-            yield { type: 'identifier', value: doubleQuoted.slice(1, -1) };
-            yield { type: 'id-quote', value: '"' };
-        }
-        else if (backtickQuoted) {
-            yield { type: 'id-quote', value: '`' };
-            yield { type: 'identifier', value: backtickQuoted.slice(1, -1) };
-            yield { type: 'id-quote', value: '`' };
-        }
-        else if (bracketQuoted) {
-            yield { type: 'id-quote', value: '[' };
-            yield { type: 'identifier', value: bracketQuoted.slice(1, -1) };
-            yield { type: 'id-quote', value: ']' };
-        }
-        else if (word) {
-            yield { type: KEYWORDS.has(word.toUpperCase()) ? 'keyword' : "identifier", value: word };
-        }
-        else if (punct) {
-            yield { type: 'punct', value: punct };
-        }
-        else if (operator) {
-            yield {type: 'operator', value: operator};
-        }
-        else if (ws) {
-            yield { type: 'whitespace', value: ws };
-        }
-        else if (unknown) {
-            yield { type: 'unknown', value: unknown };
-        }
-    }
-}
-
-export function analyzeSQLTokens(tokens: Token[]) {
-    const aliasNames = new Set<string>();
-    const tableOps: Record<string, Set<string>> = {};
-    const normalizedTokens: string[] = [];
-
-    let currentOp: { ops: string[], at: number } | null = null;
-    let lastTokenType: string | null = null;
-
-    function appendToken(val: string, type: string) {
-        if (normalizedTokens.length && type !== 'punct' && lastTokenType !== 'punct') {
-            normalizedTokens.push(' ');
-        }
-        normalizedTokens.push(val);
-        lastTokenType = type;
-    }
-
-    for (let i = 0; i < tokens.length; ) {
-        const token = tokens[i];
-
-        // Record operation context
-        if (token.type === 'keyword' && ['SELECT', 'INSERT', 'UPDATE', 'DELETE'].includes(token.value)) {
-            currentOp = { ops: [token.value.toUpperCase()], at: i };
-        }
-        else if (token.type === "keyword" && token.value === 'USING') {
-            currentOp = { ops: ["SELECT"], at: i };
-        }
-        else if (token.type === "keyword" && token.value === 'REPLACE') {
-            currentOp = { ops: ["INSERT", "UPDATE"], at: i };
-        }
-        else if (token.type === "keyword" && token.value === 'MERGE') {
-            let sawInsert = false;
-            let sawUpdate = false;
-            let sawDelete = false;
-            for (let j = i + 1; j < tokens.length; j++) {
-                if (tokens[j].type === "keyword") {
-                    if (tokens[j].value === "INSERT")
-                        sawInsert = true;
-                    if (tokens[j].value === "UPDATE")
-                        sawUpdate = true;
-                    if (tokens[j].value === "DELETE")
-                        sawDelete = true;
-                }
-            }
-            currentOp = { ops: [], at: i };
-            if (sawInsert)
-                currentOp.ops.push("INSERT");
-            if (sawUpdate)
-                currentOp.ops.push("UPDATE");
-            if (sawDelete)
-                currentOp.ops.push("DELETE");
-        }
-
-        // Detect CTE-style alias: <identifier> AS (
-        if (
-            token.type === 'identifier' &&
-            tokens[i + 1]?.type === 'keyword' &&
-            tokens[i + 1].value === 'AS' &&
-            tokens[i + 2]?.value === '('
-        ) {
-            const alias = token.value.toLowerCase();
-            aliasNames.add(alias);
-            appendToken(token.value, token.type);
-            appendToken('AS', 'keyword');
-            appendToken('(', 'punct');
-            i += 3;
-            continue;
-        }
-
-        // Detect AS <alias> (table aliases, subquery aliases, etc.)
-        if (
-            token.type === 'keyword' &&
-            token.value.toUpperCase() === 'AS' &&
-            tokens[i + 1]?.type === 'identifier'
-        ) {
-            const alias = tokens[i + 1].value.toLowerCase();
-            aliasNames.add(alias);
-            appendToken(token.value, token.type);
-            appendToken(tokens[i + 1].value, tokens[i + 1].type);
-            i += 2;
-            continue;
-        }
-
-        // Record table name if in FROM, JOIN, INTO, UPDATE
-        if (
-            token.type === 'keyword' &&
-            ['FROM', 'JOIN', 'INTO', 'UPDATE', 'USING'].includes(token.value) &&
-            tokens[i + 1]?.type === 'identifier' &&
-            !(['FROM', 'JOIN', 'USING'].includes(token.value) && tokens[i + 2]?.value === "(") // functions
-        ) {
-            const table = tokens[i + 1].value.toLowerCase();
-            if (currentOp && !aliasNames.has(table) && hasBalancedParens(tokens, currentOp.at, i)) {
-                tableOps[table] ||= new Set();
-                for (let op of currentOp.ops) {
-                    tableOps[table].add(op);
-                }
-            }
-        }
-
-        // Normalize IN (...) clauses
-        if (token.type === 'keyword' && token.value === 'IN') {
-            if (
-                tokens[i + 1]?.value === '(' &&
-                tokens[i + 2] // make sure something exists inside
-            ) {
-                appendToken('IN', 'keyword');
-                appendToken('(', 'punct');
-
-                const firstInside = tokens[i + 2];
-                if (firstInside.type === 'keyword') {
-                    // Subquery → parse normally
-                    i += 2;
-                    continue;
-                }
-                else {
-                    // Literal list → collapse
-                    appendToken('...', 'identifier');
-
-                    // Skip until matching ')'
-                    let depth = 1;
-                    let j = i + 3;
-                    while (j < tokens.length && depth > 0) {
-                        if (tokens[j].value === '(') depth++;
-                        else if (tokens[j].value === ')') depth--;
-                        j++;
-                    }
-
-                    appendToken(')', 'punct');
-                    i = j;
-                    continue;
-                }
-            }
-        }
-
-        // Normalize VALUES (...) clauses
-        if (token.type === 'keyword' && token.value === 'VALUES') {
-            if (tokens[i + 1]?.value === '(') {
-                appendToken('VALUES', 'keyword');
-                appendToken('(', 'punct');
-                appendToken('...', 'identifier');
-                appendToken(')', 'punct');
-
-                // Skip all VALUES tuples including comma-separated ones
-                let depth = 0;
-                let j = i + 1;
-                while (j < tokens.length) {
-                    if (tokens[j].value === '(') {
-                        depth++;
-                    } else if (tokens[j].value === ')') {
-                        depth--;
-                        if (depth === 0) {
-                            // Check if there's a comma after this closing paren (more tuples)
-                            let k = j + 1;
-                            while (k < tokens.length && (tokens[k].type === 'whitespace' || tokens[k].type === 'comment')) {
-                                k++;
-                            }
-                            if (tokens[k]?.value === ',') {
-                                // More tuples, continue skipping
-                                j = k + 1;
-                                continue;
-                            } else {
-                                // No more tuples, we're done
-                                break;
-                            }
-                        }
-                    }
-                    j++;
-                }
-
-                i = j + 1;
-                continue;
-            }
-        }
-
-        appendToken(token.value, token.type);
-        i++;
-    }
-
-    return {
-        tableOperations: Object.fromEntries(
-            Object.entries(tableOps).map(([k, v]) => [k, Array.from(v)])
-        ),
-        normalizedQuery: normalizedTokens.join('')
-    };
-}
-
-function hasBalancedParens(tokens: Token[], start: number, end: number): boolean {
-    let balance = 0;
-    for (let i = start; i < end; i++) {
-        const token = tokens[i];
-        if (token.type === 'punct') {
-            if (token.value === '(') balance++;
-            else if (token.value === ')') balance--;
-        }
-
-        // Early exit: unbalanced in wrong direction
-        if (balance < 0) return false;
-    }
-    return balance === 0;
-}